Are there any Dart resources that would split a command-line String into a List<String> of arguments? - shell

Are there any Dart resources that would split a command-line String into a List<String> of arguments?
ArgsParser takes a List<String> of already split arguments usually from main(List<String>).

To answer my own question,
I've converted a Java function I liked into a Dart Converter<String, List<String>) class:
import 'dart:convert';
/// Splits a `String` into a list of command-line argument parts.
/// e.g. "command -p param" -> ["command", "-p", "param"]
///
class CommandlineConverter extends Converter<String, List<String>>
{
#override
List<String> convert(String input)
{
if (input == null || input.isEmpty)
{
//no command? no string
return [];
}
final List<String> result = new List<String>();
var current = "";
String inQuote;
bool lastTokenHasBeenQuoted = false;
for (int index = 0; index < input.length; index++)
{
final token = input[index];
if (inQuote != null)
{
if (token == inQuote)
{
lastTokenHasBeenQuoted = true;
inQuote = null;
}
else
{
current += token;
}
}
else
{
switch (token)
{
case "'": // '
case '"': // ""
inQuote = token;
continue;
case " ": // space
if (lastTokenHasBeenQuoted || current.isNotEmpty)
{
result.add(current);
current = "";
}
break;
default:
current += token;
lastTokenHasBeenQuoted = false;
}
}
}
if (lastTokenHasBeenQuoted || current.isNotEmpty)
{
result.add(current);
}
if (inQuote != null)
{
throw new Exception("Unbalanced quote $inQuote in input:\n$input");
}
return result;
}
}

Related

Go through sub tags of IsoMsg and get values for jPOS

I have a jPOS project and I want to loop through sub tags of a field to get it's values. I have the following code and I don't know if it does the job:
Map<String, String> subTagMap = new HashMap<>();
ISOMsg champ11 = (ISOMsg)isoMsg.getComponent(11);
int i = 1;
while(i < 15) {
if (champ11 != null) {
ISOTaggedField subtTag = findTag(champ11, (i < 10) ? "0" + i : String.valueOf(i));
if (subtTag != null) {
LOGGER.debug("Champ11x" + i +"trouvé");
subTagMap.put(subtTag.getTag(), subtTag.getValue().toString());
}
}
i++;
}
return subTagMap;
}
private ISOTaggedField findTag(ISOComponent component, String tagName) {
if (component instanceof ISOTaggedField) {
component = ((ISOTaggedField) component).getDelegate();
}
#SuppressWarnings("unchecked")
Map<Integer, Object> children = component.getChildren();
for (Map.Entry<Integer, Object> entry : children.entrySet()) {
if (entry.getValue() instanceof ISOTaggedField) {
ISOTaggedField tag = (ISOTaggedField) entry.getValue();
if (tag.getTag().equals(tagName)) {
return tag;
}
}
}
return null;
}
This should work:
Map<String, String> subTagMap=new HashMap<>();
ISOMsg champ11 = (ISOMsg) isoMsg.getComponent(119);
if (champ != null) {
for (Object v : champ.getChildren().values()) {
if (!(v instanceof ISOTaggedField)) continue;
ISOTaggedField subTag = (ISOTaggedField) v;
subTagMap.put(subTag.getTag(), (String)subTag.getValue());
}
}
return subTagMap;

Action.Picker returns invalid/wrong Uri (How to get path or byte[] from multiple picked gallery img)

I have an forms app where i need to pick "1 to many" images from the phone storage.
For this i use the dependency injection system.
My problem is the somewhere i get an Android.netUri that resolves to a file that do not exist... and to a file name that i have never seen before.
The kicker is that if i pick pictures that was takes within the last couple of hours this code works...
Im am at the end of my hoap, i really hope someone can point me to something that i'm doing wrong.
i start the Picker activity with:
[assembly: Dependency(typeof(ImagePickerService))]
namespace MyApp.Droid
{
public class ImagePickerService : Java.Lang.Object, IImagePickerService
{
public async Task OpenGallery()
{
try
{
var status = await CrossPermissions.Current.CheckPermissionStatusAsync(Permission.Storage);
if (status != PermissionStatus.Granted)
{
if (await CrossPermissions.Current.ShouldShowRequestPermissionRationaleAsync(Permission.Storage))
{
Toast.MakeText(CrossCurrentActivity.Current.Activity, "Need Storage permission to access to your photos.", ToastLength.Long).Show();
}
var results = await CrossPermissions.Current.RequestPermissionsAsync(new[] { Permission.Storage });
status = results[Permission.Storage];
}
if (status == PermissionStatus.Granted)
{
Toast.MakeText(CrossCurrentActivity.Current.Activity, "Pick max 20 images", ToastLength.Long).Show();
var imageIntent = new Intent(Intent.ActionPick);
imageIntent.SetType("image/*");
imageIntent.PutExtra(Intent.ExtraAllowMultiple, true);
imageIntent.SetAction(Intent.ActionPick);
CrossCurrentActivity.Current.Activity.StartActivityForResult(Intent.CreateChooser(imageIntent, "Pick pictures"), 100);
}
else if (status != PermissionStatus.Unknown)
{
Toast.MakeText(CrossCurrentActivity.Current.Activity, "Permission Denied. Can not continue, try again.", ToastLength.Long).Show();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Toast.MakeText(CrossCurrentActivity.Current.Activity, "Error. Can not continue, try again.", ToastLength.Long).Show();
}
}
}
then in my MainActivity.cs i have the OnActivityResult
I have tried to use the ContentResolver.OpenInputStream to get the image bytes with no luck, so this is commented out atm.
protected override void OnActivityResult(int requestCode, Result resultCode, Intent data)
{
base.OnActivityResult(requestCode, resultCode, data);
if (requestCode == OPENGALLERYCODE && resultCode == Result.Ok)
{
List<string> images = new List<string>();
if (data != null)
{
ClipData clipData = data.ClipData;
if (clipData != null)
{
for (int i = 0; i < clipData.ItemCount; i++)
{
ClipData.Item item = clipData.GetItemAt(i);
/*
var stream = ContentResolver.OpenInputStream(item.Uri); //This throws "FileNotFound"
byte[] byteArray;
using (var memoryStream = new MemoryStream())
{
stream.CopyTo(memoryStream);
byteArray = memoryStream.ToArray();
stream.Close();
stream = null;
}
stream = ContentResolver.OpenInputStream(item.Uri);
var exif = new ExifInterface(stream);
stream.Close();
*/
Android.Net.Uri uri = item.Uri;
var path = GetActualPathFromFile(uri);
if (path != null)
{
var tmpImgPath = RotateToOriginalDimention(path);
images.Add(tmpImgPath);
}
}
}
else
{
Android.Net.Uri uri = data.Data;
var path = GetActualPathFromFile(uri);
if (path != null)
{
var tmpImgPath = RotateToOriginalDimention(path);
images.Add(tmpImgPath);
}
}
MessagingCenter.Send<App, List<string>>((App)Xamarin.Forms.Application.Current, "ImagesSelected", images);
}
}
}
And the GetActualPathFromFile (also in my MainActivity.cs)
The hole func is below but i hit this part of the code and get at "FileNotFound"
(...)
else if ("content".Equals(uri.Scheme, StringComparison.OrdinalIgnoreCase))
{
var retval2 = getDataColumn(this, uri, null, null);
if (File.Exists(retval2)) //<----------------------- This returns "false"
{
return retval2;
}
else
{
throw new Exception("file not found " + retval2);
}
}
(...)
The Hole GetActualPathFromFile
private string GetActualPathFromFile(Android.Net.Uri uri)
{
bool isKitKat = Build.VERSION.SdkInt >= Android.OS.BuildVersionCodes.Kitkat;
if (isKitKat && DocumentsContract.IsDocumentUri(this, uri))
{
// ExternalStorageProvider
if (isExternalStorageDocument(uri))
{
string docId = DocumentsContract.GetDocumentId(uri);
char[] chars = { ':' };
string[] split = docId.Split(chars);
string type = split[0];
if ("primary".Equals(type, StringComparison.OrdinalIgnoreCase))
{
var retval = Android.OS.Environment.ExternalStorageDirectory + "/" + split[1];
if (File.Exists(retval))
{
return retval;
}
else
{
throw new Exception("file not found " + retval);
}
}
}
// DownloadsProvider
else if (isDownloadsDocument(uri))
{
string id = DocumentsContract.GetDocumentId(uri);
Android.Net.Uri contentUri = ContentUris.WithAppendedId(
Android.Net.Uri.Parse("content://downloads/public_downloads"), long.Parse(id));
//System.Diagnostics.Debug.WriteLine(contentUri.ToString());
var retval = getDataColumn(this, contentUri, null, null);
if (File.Exists(retval))
{
return retval;
}
else
{
throw new Exception("file not found " + retval);
}
}
// MediaProvider
else if (isMediaDocument(uri))
{
String docId = DocumentsContract.GetDocumentId(uri);
char[] chars = { ':' };
String[] split = docId.Split(chars);
String type = split[0];
Android.Net.Uri contentUri = null;
if ("image".Equals(type))
{
contentUri = MediaStore.Images.Media.ExternalContentUri;
}
else if ("video".Equals(type))
{
contentUri = MediaStore.Video.Media.ExternalContentUri;
}
else if ("audio".Equals(type))
{
contentUri = MediaStore.Audio.Media.ExternalContentUri;
}
String selection = "_id=?";
String[] selectionArgs = new String[]
{
split[1]
};
var retval = getDataColumn(this, contentUri, selection, selectionArgs);
if (File.Exists(retval))
{
return retval;
}
else
{
throw new Exception("file not found " + retval);
}
}
}
// MediaStore (and general)
else if ("content".Equals(uri.Scheme, StringComparison.OrdinalIgnoreCase))
{
// Return the remote address
if (isGooglePhotosUri(uri))
{
var retval = uri.LastPathSegment;
if (File.Exists(retval))
{
return retval;
}
else
{
throw new Exception("file not found " + retval);
}
}
var retval2 = getDataColumn(this, uri, null, null);
if (File.Exists(retval2))
{
return retval2;
}
else
{
throw new Exception("file not found " + retval2);
}
}
// File
else if ("file".Equals(uri.Scheme, StringComparison.OrdinalIgnoreCase))
{
var retval = uri.Path;
if (File.Exists(retval))
{
return retval;
}
else
{
throw new Exception("file not found " + retval);
}
}
throw new Exception("file not found ");
}
public static String getDataColumn(Context context, Android.Net.Uri uri, String selection, String[] selectionArgs)
{
ICursor cursor = null;
String column = "_data";
String[] projection =
{
column
};
try
{
cursor = context.ContentResolver.Query(uri, projection, selection, selectionArgs, null);
if (cursor != null && cursor.MoveToFirst())
{
int index = cursor.GetColumnIndexOrThrow(column);
return cursor.GetString(index);
}
}
finally
{
if (cursor != null)
cursor.Close();
}
return null;
}
//Whether the Uri authority is ExternalStorageProvider.
public static bool isExternalStorageDocument(Android.Net.Uri uri)
{
return "com.android.externalstorage.documents".Equals(uri.Authority);
}
//Whether the Uri authority is DownloadsProvider.
public static bool isDownloadsDocument(Android.Net.Uri uri)
{
return "com.android.providers.downloads.documents".Equals(uri.Authority);
}
//Whether the Uri authority is MediaProvider.
public static bool isMediaDocument(Android.Net.Uri uri)
{
return "com.android.providers.media.documents".Equals(uri.Authority);
}
//Whether the Uri authority is Google Photos.
public static bool isGooglePhotosUri(Android.Net.Uri uri)
{
return "com.google.android.apps.photos.content".Equals(uri.Authority);
}
Found out that the real problem was that Google Photos App was not updating and was still showing images that were deleted.
After 2x reboot of the phone, Google Photos app finally updated.
So this looks more like a cache problem with Google Foto than a xamarin problem.

how to exclude checking words in quotations using java

It's the first time I'm doing this so I didn't want to be lengthy. I'm building a cross reference from reading a java program. I'm to exclude java keywords, commented words and words in quotations. I got through with excluding the java keywords and the commented words but I'm having problems excluding those in quotes.
public class CrossReference {
static Scanner in;
static PrintWriter out;
static int currentLine = 0;
public static void main(String[] args) throws IOException {
in = new Scanner (new FileReader("keywords.txt"));
out = new PrintWriter (new FileWriter("crossreference.out"));
LinkedList keywords = new LinkedList();
while (in.hasNextLine()) {
String word = in.nextLine();
keywords.addTail(new NodeData(word));
}
in = new Scanner (new FileReader("program.txt"));
BinaryTree bst = new BinaryTree();
while(in.hasNextLine()){
String line = in.nextLine();
out.printf("%3d. %s\n", ++currentLine,line);
getWordsOnLine(line,bst,keywords);
}
out.printf("\nWords LineNumber\n\n");
bst.inOrder();
out.close();
}
public static void getWordsOnLine(String inputLine, BinaryTree bst, LinkedList keywords){
Scanner inLine = new Scanner(inputLine);
inLine.useDelimiter("[^a-zA-Z//\"*]+");
boolean b = true;
while(inLine.hasNext() && b){
String word = inLine.next().toLowerCase();
if (word.contains("/") || word.contains("\"") || word.contains("*")) {
b = false;
} //this works for the commented words but not so well for the ones in quotes as it also excludes words after those in quotes
else {
boolean key = false;
Node curr = keywords.head;
while (curr != null) {
if (curr.data.str.equals(word)) key = true;
curr = curr.next;
}
if (key == false) {
TreeNode node = bst.findOrInsert(new TreeNodeData(word));
ListNode p = new ListNode(currentLine);
p.next = node.data.firstLine;
node.data.firstLine = p;
}
}
}
}
}
Split your string by space into an array.
Iterate through the array, checking which elements start with quotes and are words
Sum it
So the code would look like:
public class HelloWorld
{
public static void main(String[] args)
{
String a = "COPY PASTE ORIGINAL HERE";
String[] arr = a.split(" ");
int count = 0;
for(String each: arr){
if(each.charAt(0) != '\"' && each.charAt(0) < '0' || each.charAt(0) > '9'){
count++;
}
}
System.out.println("words="+count);
}
}

How to translate Linq Expression where clause to hql where clause?

For some reason, I need to combine Linq Expression (only where clause) & an HQL where clause into one query.
I find that the session.Query<T>() API will translate Linq Expression to a HqlQuery object (that extends HqlExpression).
How can I translate the Linq Expression where clause to an HQL where clause queryString, and then I can combine another HQL where clause queryString into a new query?
Seems that is not possible to use exists NHibernate API to convert Linq expression to HQL tree.
The HQL tree produced from the Linq expression is not reversable to an actual HQL query.
So I have to translate Linq expression to HQL by self:
var expr = GetExpr<Ninja>(x =>
x.Age > 1 && x.Country.Name == "中国"
||
(x.Id > 10 && x.Country.Name == "中国")
);
var translator = new ExpressionToHqlTranslator("_this");
translator.Translate(expr);
Console.WriteLine(translator.WhereClause);
Console.WriteLine(translator.Patameters);
============== result =============
WhereClause: (((_this.Age > ?) AND (_this.Country.Name = ?)) OR ((_this.Id > ?) AND (_this.Country.Name = ?)))
Patameters:4
=============== the critical code =============
static Expression<Func<T, object>> GetExpr<T>(Expression<Func<T, object>> expr){
eturn expr;
}
using System;
using System.Linq;
using NHibernate.Linq;
using NHibernate.Linq.Visitors;
using System.Linq.Expressions;
using NHibernate;
using System.Text;
using System.Collections.Generic;
namespace Rhythm.Linq
{
public class ExpressionToHqlTranslator : System.Linq.Expressions.ExpressionVisitor
{
private StringBuilder sb;
private string _orderBy = "";
private int? _skip = null;
private int? _take = null;
private string _whereClause = "";
List<object> patameters;
public int? Skip
{
get
{
return _skip;
}
}
public int? Take
{
get
{
return _take;
}
}
public string OrderBy
{
get
{
return _orderBy;
}
}
public string WhereClause
{
get
{
return _whereClause;
}
}
public List<object> Patameters
{
get
{
return patameters;
}
set
{
patameters = value;
}
}
string prefix;
public ExpressionToHqlTranslator(string prefix = null)
{
this.prefix = string.IsNullOrEmpty(prefix) ? null : (prefix + ".");
}
public string Translate(Expression expression)
{
this.sb = new StringBuilder();
this.patameters = new List<object>();
this.Visit(expression);
_whereClause = this.sb.ToString();
return _whereClause;
}
private static Expression StripQuotes(Expression e)
{
while (e.NodeType == ExpressionType.Quote)
{
e = ((UnaryExpression)e).Operand;
}
return e;
}
protected override Expression VisitMethodCall(MethodCallExpression m)
{
if (m.Method.DeclaringType == typeof(Queryable) && m.Method.Name == "Where")
{
this.Visit(m.Arguments[0]);
LambdaExpression lambda = (LambdaExpression)StripQuotes(m.Arguments[1]);
this.Visit(lambda.Body);
return m;
}
else if (m.Method.Name == "Take")
{
if (this.ParseTakeExpression(m))
{
Expression nextExpression = m.Arguments[0];
return this.Visit(nextExpression);
}
}
else if (m.Method.Name == "Skip")
{
if (this.ParseSkipExpression(m))
{
Expression nextExpression = m.Arguments[0];
return this.Visit(nextExpression);
}
}
else if (m.Method.Name == "OrderBy")
{
if (this.ParseOrderByExpression(m, "ASC"))
{
Expression nextExpression = m.Arguments[0];
return this.Visit(nextExpression);
}
}
else if (m.Method.Name == "OrderByDescending")
{
if (this.ParseOrderByExpression(m, "DESC"))
{
Expression nextExpression = m.Arguments[0];
return this.Visit(nextExpression);
}
}
throw new NotSupportedException(string.Format("The method '{0}' is not supported", m.Method.Name));
}
protected override Expression VisitUnary(UnaryExpression u)
{
switch (u.NodeType)
{
case ExpressionType.Not:
sb.Append(" NOT ");
this.Visit(u.Operand);
break;
case ExpressionType.Convert:
this.Visit(u.Operand);
break;
default:
throw new NotSupportedException(string.Format("The unary operator '{0}' is not supported", u.NodeType));
}
return u;
}
/// <summary>
///
/// </summary>
/// <param name="b"></param>
/// <returns></returns>
protected override Expression VisitBinary(BinaryExpression b)
{
sb.Append("(");
this.Visit(b.Left);
switch (b.NodeType)
{
case ExpressionType.And:
sb.Append(" AND ");
break;
case ExpressionType.AndAlso:
sb.Append(" AND ");
break;
case ExpressionType.Or:
sb.Append(" OR ");
break;
case ExpressionType.OrElse:
sb.Append(" OR ");
break;
case ExpressionType.Equal:
if (IsNullConstant(b.Right))
{
sb.Append(" IS ");
}
else
{
sb.Append(" = ");
}
break;
case ExpressionType.NotEqual:
if (IsNullConstant(b.Right))
{
sb.Append(" IS NOT ");
}
else
{
sb.Append(" <> ");
}
break;
case ExpressionType.LessThan:
sb.Append(" < ");
break;
case ExpressionType.LessThanOrEqual:
sb.Append(" <= ");
break;
case ExpressionType.GreaterThan:
sb.Append(" > ");
break;
case ExpressionType.GreaterThanOrEqual:
sb.Append(" >= ");
break;
default:
throw new NotSupportedException(string.Format("The binary operator '{0}' is not supported", b.NodeType));
}
this.Visit(b.Right);
sb.Append(")");
return b;
}
protected override Expression VisitConstant(ConstantExpression c)
{
this.patameters.Add(c.Value);
sb.Append('?');
//IQueryable q = c.Value as IQueryable;
//if (q == null && c.Value == null)
//{
// sb.Append("NULL");
//}
//else if (q == null)
//{
// switch (Type.GetTypeCode(c.Value.GetType()))
// {
// case TypeCode.Boolean:
// sb.Append(((bool)c.Value) ? 1 : 0);
// break;
// case TypeCode.String:
// sb.Append("'");
// sb.Append(c.Value);
// sb.Append("'");
// break;
// case TypeCode.DateTime:
// sb.Append("'");
// sb.Append(c.Value);
// sb.Append("'");
// break;
// case TypeCode.Object:
// throw new NotSupportedException(string.Format("The constant for '{0}' is not supported", c.Value));
// default:
// sb.Append(c.Value);
// break;
// }
//}
return c;
}
protected override Expression VisitMember(MemberExpression m)
{
if (this.prefix != null)
{
sb.Append(this.prefix);
}
sb.Append(ContactModelPropertyVistHierarchyExpression(m, m.Member.DeclaringType));
//if (m.Expression != null && m.Expression.NodeType == ExpressionType.Parameter)
//{
// sb.Append(m.Member.Name);
// return m;
//}
return m;
//throw new NotSupportedException(string.Format("The member '{0}' is not supported", m.Member.Name));
}
protected bool IsNullConstant(Expression exp)
{
return (exp.NodeType == ExpressionType.Constant && ((ConstantExpression)exp).Value == null);
}
private bool ParseOrderByExpression(MethodCallExpression expression, string order)
{
UnaryExpression unary = (UnaryExpression)expression.Arguments[1];
LambdaExpression lambdaExpression = (LambdaExpression)unary.Operand;
lambdaExpression = (LambdaExpression)NHibernate.Linq.Visitors.Evaluator.PartialEval(lambdaExpression);
MemberExpression body = lambdaExpression.Body as MemberExpression;
if (body != null)
{
if (string.IsNullOrEmpty(_orderBy))
{
_orderBy = string.Format("{0} {1}", body.Member.Name, order);
}
else
{
_orderBy = string.Format("{0}, {1} {2}", _orderBy, body.Member.Name, order);
}
return true;
}
return false;
}
private bool ParseTakeExpression(MethodCallExpression expression)
{
ConstantExpression sizeExpression = (ConstantExpression)expression.Arguments[1];
int size;
if (int.TryParse(sizeExpression.Value.ToString(), out size))
{
_take = size;
return true;
}
return false;
}
private bool ParseSkipExpression(MethodCallExpression expression)
{
ConstantExpression sizeExpression = (ConstantExpression)expression.Arguments[1];
int size;
if (int.TryParse(sizeExpression.Value.ToString(), out size))
{
_skip = size;
return true;
}
return false;
}
}
public static string ContactModelPropertyVistHierarchyExpression(Expression expr, Type modelType)
{
StringBuilder sb = new StringBuilder();
Expression curr = expr;
// TypedParameterExpression
while (curr != null)
{
if (curr is MemberExpression)
{
var x = curr as MemberExpression;
sb.Insert(0, x.Member.Name);
curr = x.Expression;
}
else if (curr is MethodCallExpression)
{
var x = curr as MethodCallExpression;
sb.Insert(0, x.Method.Name);
curr = x.Object;
}
else if (curr is ParameterExpression)
{
break;
}
else
{
throw new ArgumentException("Unsupported Expression type " + curr.GetType().FullName + " for expression " + expr.ToString(), "expr");
}
sb.Insert(0, '.');
}
return sb.Length > 1 ? sb.Remove(0, 1).ToString() : sb.ToString();
}
}
add dll reference NHibernate.Linq.dll

Reading a file with newlines as a tuple in pig

Is it possible to change the record delimiter from newline to some other string so as to read a file with newlines into a single tuple in pig.
Yes.
A = LOAD '...' USING PigStorage(',') AS (...); //comma is the delimeter for fields
SET textinputformat.record.delimiter '<delimeter>'; // record delimeter, by default it is `\n`. You can change to any delimeter.
As mentioned here
You can use PigStorage
A = LOAD '/some/path/COMMA-DELIM-PREFIX*' USING PigStorage(',') AS (f1:chararray, ...);
B = LOAD '/some/path/SEMICOLON-DELIM-PREFIX*' USING PigStorage('\t') AS (f1:chararray, ...);
You can even try writing load/store UDF.
There is java code example for both load and store.
Load Functions : LoadFunc abstract class has the main methods for loading data and for most use cases it would suffice to extend it. You can read more here
Example
The loader implementation in the example is a loader for text data
with line delimiter as '\n' and '\t' as default field delimiter (which
can be overridden by passing a different field delimiter in the
constructor) - this is similar to current PigStorage loader in Pig.
The implementation uses an existing Hadoop supported Inputformat -
TextInputFormat - as the underlying InputFormat.
public class SimpleTextLoader extends LoadFunc {
protected RecordReader in = null;
private byte fieldDel = '\t';
private ArrayList<Object> mProtoTuple = null;
private TupleFactory mTupleFactory = TupleFactory.getInstance();
private static final int BUFFER_SIZE = 1024;
public SimpleTextLoader() {
}
/**
* Constructs a Pig loader that uses specified character as a field delimiter.
*
* #param delimiter
* the single byte character that is used to separate fields.
* ("\t" is the default.)
*/
public SimpleTextLoader(String delimiter) {
this();
if (delimiter.length() == 1) {
this.fieldDel = (byte)delimiter.charAt(0);
} else if (delimiter.length() > 1 & & delimiter.charAt(0) == '\\') {
switch (delimiter.charAt(1)) {
case 't':
this.fieldDel = (byte)'\t';
break;
case 'x':
fieldDel =
Integer.valueOf(delimiter.substring(2), 16).byteValue();
break;
case 'u':
this.fieldDel =
Integer.valueOf(delimiter.substring(2)).byteValue();
break;
default:
throw new RuntimeException("Unknown delimiter " + delimiter);
}
} else {
throw new RuntimeException("PigStorage delimeter must be a single character");
}
}
#Override
public Tuple getNext() throws IOException {
try {
boolean notDone = in.nextKeyValue();
if (!notDone) {
return null;
}
Text value = (Text) in.getCurrentValue();
byte[] buf = value.getBytes();
int len = value.getLength();
int start = 0;
for (int i = 0; i < len; i++) {
if (buf[i] == fieldDel) {
readField(buf, start, i);
start = i + 1;
}
}
// pick up the last field
readField(buf, start, len);
Tuple t = mTupleFactory.newTupleNoCopy(mProtoTuple);
mProtoTuple = null;
return t;
} catch (InterruptedException e) {
int errCode = 6018;
String errMsg = "Error while reading input";
throw new ExecException(errMsg, errCode,
PigException.REMOTE_ENVIRONMENT, e);
}
}
private void readField(byte[] buf, int start, int end) {
if (mProtoTuple == null) {
mProtoTuple = new ArrayList<Object>();
}
if (start == end) {
// NULL value
mProtoTuple.add(null);
} else {
mProtoTuple.add(new DataByteArray(buf, start, end));
}
}
#Override
public InputFormat getInputFormat() {
return new TextInputFormat();
}
#Override
public void prepareToRead(RecordReader reader, PigSplit split) {
in = reader;
}
#Override
public void setLocation(String location, Job job)
throws IOException {
FileInputFormat.setInputPaths(job, location);
}
}
Store Functions : StoreFunc abstract class has the main methods for storing data and for most use cases it should suffice to extend it
Example
The storer implementation in the example is a storer for text data
with line delimiter as '\n' and '\t' as default field delimiter (which
can be overridden by passing a different field delimiter in the
constructor) - this is similar to current PigStorage storer in Pig.
The implementation uses an existing Hadoop supported OutputFormat -
TextOutputFormat as the underlying OutputFormat.
public class SimpleTextStorer extends StoreFunc {
protected RecordWriter writer = null;
private byte fieldDel = '\t';
private static final int BUFFER_SIZE = 1024;
private static final String UTF8 = "UTF-8";
public PigStorage() {
}
public PigStorage(String delimiter) {
this();
if (delimiter.length() == 1) {
this.fieldDel = (byte)delimiter.charAt(0);
} else if (delimiter.length() > 1delimiter.charAt(0) == '\\') {
switch (delimiter.charAt(1)) {
case 't':
this.fieldDel = (byte)'\t';
break;
case 'x':
fieldDel =
Integer.valueOf(delimiter.substring(2), 16).byteValue();
break;
case 'u':
this.fieldDel =
Integer.valueOf(delimiter.substring(2)).byteValue();
break;
default:
throw new RuntimeException("Unknown delimiter " + delimiter);
}
} else {
throw new RuntimeException("PigStorage delimeter must be a single character");
}
}
ByteArrayOutputStream mOut = new ByteArrayOutputStream(BUFFER_SIZE);
#Override
public void putNext(Tuple f) throws IOException {
int sz = f.size();
for (int i = 0; i < sz; i++) {
Object field;
try {
field = f.get(i);
} catch (ExecException ee) {
throw ee;
}
putField(field);
if (i != sz - 1) {
mOut.write(fieldDel);
}
}
Text text = new Text(mOut.toByteArray());
try {
writer.write(null, text);
mOut.reset();
} catch (InterruptedException e) {
throw new IOException(e);
}
}
#SuppressWarnings("unchecked")
private void putField(Object field) throws IOException {
//string constants for each delimiter
String tupleBeginDelim = "(";
String tupleEndDelim = ")";
String bagBeginDelim = "{";
String bagEndDelim = "}";
String mapBeginDelim = "[";
String mapEndDelim = "]";
String fieldDelim = ",";
String mapKeyValueDelim = "#";
switch (DataType.findType(field)) {
case DataType.NULL:
break; // just leave it empty
case DataType.BOOLEAN:
mOut.write(((Boolean)field).toString().getBytes());
break;
case DataType.INTEGER:
mOut.write(((Integer)field).toString().getBytes());
break;
case DataType.LONG:
mOut.write(((Long)field).toString().getBytes());
break;
case DataType.FLOAT:
mOut.write(((Float)field).toString().getBytes());
break;
case DataType.DOUBLE:
mOut.write(((Double)field).toString().getBytes());
break;
case DataType.BYTEARRAY: {
byte[] b = ((DataByteArray)field).get();
mOut.write(b, 0, b.length);
break;
}
case DataType.CHARARRAY:
// oddly enough, writeBytes writes a string
mOut.write(((String)field).getBytes(UTF8));
break;
case DataType.MAP:
boolean mapHasNext = false;
Map<String, Object> m = (Map<String, Object>)field;
mOut.write(mapBeginDelim.getBytes(UTF8));
for(Map.Entry<String, Object> e: m.entrySet()) {
if(mapHasNext) {
mOut.write(fieldDelim.getBytes(UTF8));
} else {
mapHasNext = true;
}
putField(e.getKey());
mOut.write(mapKeyValueDelim.getBytes(UTF8));
putField(e.getValue());
}
mOut.write(mapEndDelim.getBytes(UTF8));
break;
case DataType.TUPLE:
boolean tupleHasNext = false;
Tuple t = (Tuple)field;
mOut.write(tupleBeginDelim.getBytes(UTF8));
for(int i = 0; i < t.size(); ++i) {
if(tupleHasNext) {
mOut.write(fieldDelim.getBytes(UTF8));
} else {
tupleHasNext = true;
}
try {
putField(t.get(i));
} catch (ExecException ee) {
throw ee;
}
}
mOut.write(tupleEndDelim.getBytes(UTF8));
break;
case DataType.BAG:
boolean bagHasNext = false;
mOut.write(bagBeginDelim.getBytes(UTF8));
Iterator<Tuple> tupleIter = ((DataBag)field).iterator();
while(tupleIter.hasNext()) {
if(bagHasNext) {
mOut.write(fieldDelim.getBytes(UTF8));
} else {
bagHasNext = true;
}
putField((Object)tupleIter.next());
}
mOut.write(bagEndDelim.getBytes(UTF8));
break;
default: {
int errCode = 2108;
String msg = "Could not determine data type of field: " + field;
throw new ExecException(msg, errCode, PigException.BUG);
}
}
}
#Override
public OutputFormat getOutputFormat() {
return new TextOutputFormat<WritableComparable, Text>();
}
#Override
public void prepareToWrite(RecordWriter writer) {
this.writer = writer;
}
#Override
public void setStoreLocation(String location, Job job) throws IOException {
job.getConfiguration().set("mapred.textoutputformat.separator", "");
FileOutputFormat.setOutputPath(job, new Path(location));
if (location.endsWith(".bz2")) {
FileOutputFormat.setCompressOutput(job, true);
FileOutputFormat.setOutputCompressorClass(job, BZip2Codec.class);
} else if (location.endsWith(".gz")) {
FileOutputFormat.setCompressOutput(job, true);
FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
}
}
}

Resources