I have setup like this:
#read a list of files
tr -d \\015 < sample.txt | while IFS=, read -r NAME
do
#grep for lines and do stuff
for VAR in $(grep '.*: {$' $NAME)
do
do some stuff
done
done
Problem is this. It doesn't work, because the for VAR in $(grep '.*: {$' $NAME) adds an unnecessary space and newline to its results.
If I echo $VAR I get the following:
blahblahblah:
{
Now consider this code:
#read a list of files
tr -d \\015 < sample.txt | while IFS=, read -r NAME
do
VAR=$(grep '.*: {$' $NAME)
echo $VAR
done
If I echo $VAR here I get:
blahblahblah: {
Why do I get the extra newline in the first example?
Related
I'm trying to locate a list of strings from a .txt file, the search target is a directory of multiple .csv (locating which .csv contain the string)
I already find how to do it manually:
grep -rl doggo C:\dirofcsv\
The next step is to to it from a list of hundreds of terms.
I tried grep -rl -f list.txt C:\dirofcsv < print.txt but I only have the last term printed.. I want to have the results lines by lines.
I'm missing something but I don't know where.
I'm working on windows with a term emulator.
EDIT: I've found how to list the terms from a file.Now I need to see which terms have which result like " doggo => file2, file4" did I need to write a loop ?
Thanks community.
grep -rl -f list.txt C:\dirofcsv >> print.txt
You are looking to append lines to the print.txt file and so will need to use >> as opposed to > which will overwrite what is already in the file.
To get the output listed in the output required in your edited requirement, you can use a loop redirected back into awk:
awk '/^FILE -/ { fil=$3; # When the output start with "FILE -" set fil to the third space delimited field
next # Skip to the next line
}
{ arr[fil][$0]="" # Set up a 2 dimensional array with the search term (fil) as the first index and the name of the file the second
}
END { for (i in arr) { # Loop through the array
printf "%s => ",i; First print the search term in the format required
for (j in arr[i]) {
printf "%s,",j # Print the file name followed by a comma
}
printf "\n" # Print a new line
}
}' <<< "$(while read line # Read list.txt line by line
do
echo "FILE - $line"; Echo a marker for identification in awk
grep -l "$line" C:\dirofcsv ; # Grep for the line
done < list.txt)" >> print.txt
One liner:
awk '/^FILE -/ { fil=$3;next } { arr[fil][$0]="" } END { for (i in arr) { printf "%s => ",i;for (j in arr[i]) { printf "%s,",j } printf "\n" } }' <<< "$(while read line;do echo "FILE - $line";grep -l "$line" C:\dirofcsv done < list.txt)" >> print.txt
I think you meant to pass the command as:
grep -rl -f list.txt C:\dirofcsv >> print.txt
Give it a shot. It should take all patterns from list.txt line by line and search in the directory C:\dirofcsv for files with matching patterns and print their names to print.txt file.
Try this for printing without a loop (just like you asked in comments ;-)
One Line Answer
dir=C:\dirofcsv
listfile=list.txt
eval $(jq -Rsr 'split("\n") | map(select(length > 0)) | reduce .[] as $line ([]; . + ["echo \($line) :; grep -rl \($line) \($dir); echo"]) | (join("; "))' --arg dir "$dir" < "$listfile")
Another solution, for explanation say:
unset li
readarray li -u <"$listfile"
quoted_commands="$(jq -R 'reduce inputs as $line ([]; . + ["echo \($line) :; grep -rl \($line) \($dir); echo"]) | (join("; "))' \
--arg dir $dir \
<<< $(echo; printf "%s" "${li[#]}"))"
quoted_commands=${quoted_commands%\"}
commands=${quoted_commands#\"}
eval $commands
Breaking down the command for better explaination in comments:
# read contents of listfile in li
unset li && readarray li -u <"$listfile"
# add the content to new list so that it prints the list elements in new-lines
# also add a newline at top as it will be discarded by jq (in this case only)
list="$(echo; printf "%s" "${li[#]}";)"
# pass jq command
quoted_commands="$(jq -R 'reduce inputs as $line
([]; . + ["echo \($line) :; grep -rl \($line) \($dir); echo"])
| (join("; "))' \
--arg dir $dir <<< "$list")"
# the elements are read with reduce filter and converted to JSON Array of corresponding commands to execute
# the commands for all elements of list are joined with join filter
# trim quotes to execute commands properly
commands=$(sed -e 's/^"//' -e 's/"$//' <<< "$quoted_commands")
# run commands
eval "$commands"
You may want to print the above variables. Take care to use quotes in echo/printf while doing so, i.e., echo "$variable".
Replacement of sed command:
signgle_quoted=${quoted%\"}
commands=${signgle_quoted#\"}
echo "$commands"
I am now using the following implementations (though the dictionary implementation uses a for loop, the key : value implementation doesn't, and is a single line command):
# print an Associative bash array as a JSON dictionary
print_dict()
{
declare -n ref
ref=$1
for k in $(echo "${!ref[#]}")
do
printf '{"name":"%s", "value":"%s"}\n' "$k" "${ref[$k]}"
done | jq -s 'reduce .[] as $i ({}; .[$i.name] = $i.value)'
}
#-------------------------------------------------------------------------
# print the grep output in key : value format
function list_grep()
{
local listfile=$1
local dir=$2
eval $(jq -Rsr 'split("\n") | map(select(length > 0)) | reduce .[] as $line ([]; . + ["echo \($line) :; grep -rl \($line) \($dir); echo"]) | (join("; "))' --arg dir "$dir" < "$listfile")
}
#-------------------------------------------------------------------------
# print the grep output as JSON dictionary
function dict_grep()
{
local listfile=$1
local dir=$2
eval declare -A Arr=\($(eval echo $(jq -Rrs 'split("\n") | map(select(length > 0)) | reduce .[] as $k ([]; . + ["[\($k)]=\\\"$(grep -rl \($k) tmp)\\\""]) | (join(" "))' --arg dir $dir < tmp/list.txt))\)
print_dict Arr
}
#-------------------------------------------------------------------------
# call:
list_grep $listfile $dir
dict_grep $listfile $dir
-Himanshu
I'm trying to decode a file, which is mostly encoded with base64. What I want to do is to decode the following, while still maintaining the [_*_].
example.txt
wq9cXyjjg4QpXy/Crwo=
[_NOTBASE64ED_]
aGkgdGhlcmUK
[_CONSTANT_]
SGVsbG8gV29ybGQhCg==
Sometimes it'll be in this form
aGkgdGhlcmUK[_CONSTANT_]SGVsbG8gV29ybGQhCg==
Desired output
¯\_(ツ)_/¯
[_NOTBASE64ED_]
hi there
[_CONSTANT_]
Hello World!
hi there[_CONSTANT_]Hello World!
Error output
¯\_(ツ)_/¯
4��!:�#�H\�B�8ԓ��[��ܛBbase64: invalid input
What I've tried
base64 -di example.txt
base64 -d example.txt
base64 --wrap=0 -d -i example.txt
I tried to individually base64 the [_*_] using grep -o. Then find and
replacing them through a weird arrangement with arrays, but I couldn't
get it to work.
base64ing it all, then decoding. Results in double base64ed rows.
The file is significantly downsized!
Encoded using base64 --wrap=0, while loop, and if/else statement.
The [_*_] still need to be there after being decoded.
I am sure someone has a more clever solution than this. But try this
#! /bin/bash
MYTMP1=""
function printInlineB64()
{
local lines=($(echo $1 | sed -e 's/\[/\n[/g' -e 's/\]/]\n/g'))
OUTPUT=""
for line in "${lines[#]}"; do
MYTMP1=$(base64 -d <<< "$line" 2>/dev/null)
if [ "$?" != "0" ]; then
OUTPUT="${OUTPUT}${line}"
else
OUTPUT="${OUTPUT}${MYTMP1}"
fi;
done
echo "$OUTPUT"
}
MYTMP2=""
function printB64Line()
{
local line=$1
# not fully base64 line
if [[ ! "$line" =~ ^[A-Za-z0-9+/=]+$ ]]; then
printInlineB64 "$line"
return
fi;
# likely base64 line
MYTMP2=$(base64 -d <<< "$line" 2>/dev/null)
if [ "$?" != "0" ]; then
echo $line
else
echo $MYTMP2
fi;
}
FILE=$1
if [ -z "$FILE" ]; then
echo "Please give a file name in argument"
exit 1;
fi;
while read line; do
printB64Line "$line"
done < ${FILE}
and here is output
$ cat example.txt && echo "==========================" && ./base64.sh example.txt
wq9cXyjjg4QpXy/Crwo=
[_NOTBASE64ED_]
aGkgdGhlcmUK
[_CONSTANT_]
SGVsbG8gV29ybGQhCg==
==========================
¯\_(ツ)_/¯
[_NOTBASE64ED_]
hi there
[_CONSTANT_]
Hello World!
$ cat example2.txt && echo "==========================" && ./base64.sh example2.txt
aGkgdGhlcmUK[_CONSTANT_]SGVsbG8gV29ybGQhCg==
==========================
hi there[_CONSTANT_]Hello World!
You need a loop that reads each line and tests whether it's base64 or non-base64, and processes it appropriately.
while read -r line
do
case "$line" in
\[*\]) echo "$line" ;;
*) base64 -d <<< "$line" ;;
esac
done << example.txt
I would suggest using other languages other than sh but here is a solution using cut. This would handle the case where there are more than one [_constant_] in a line.
#!/bin/bash
function decode() {
local data=""
local line=$1
while [[ -n $line ]]; do
data=$data$(echo $line | cut -d[ -f1 | base64 -d)
const=$(echo $line | cut -d[ -sf2- | cut -d] -sf1)
[[ -n $const ]] && data=$data[$const]
line=$(echo $line | cut -d] -sf2-)
done
echo "$data"
}
while read -r line; do
decode $line
done < example.txt
If Perl is an option, you can say something like:
perl -MMIME::Base64 -lpe '$_ = join("", grep {/^\[/ || chomp($_ = decode_base64($_)), 1} split(/(?=\[)|(?<=\])/))' example.txt
The code below is equivalent to the above but is broken down into steps for the explanation purpose:
#!/bin/bash
perl -MMIME::Base64 -lpe '
#ary = split(/(?=\[)|(?<=\])/, $_);
foreach (#ary) {
if (! /^\[/) {
chomp($_ = decode_base64($_));
}
}
$_ = join("", #ary);
' example.txt
-MMIME::Base64 option loads the base64 codec module.
-lpe option makes Perl bahave like AWK to loop over input lines and implicitly handle newlines.
The regular expression (?=\[)|(?<=\]) matches the boundary between the base64 block and the maintaining block surrounded by [...].
The split function divides the line into blocks on the boundary and store them in an array.
Then loop over the array and decode the base64-encoded entry if found.
Finally merge the substring blocks into a line to print.
I am reading in from a .txt file which looks something like this:
:DRIVES
name,server,share_1
other_name,other_server,share_2
new_name,new_server,share 3
:NAME
which is information to mount drives. I want to load them into a bash array to cycle through and mount them, however my current code breaks at the third line because the array is being created by any white space. Instead of reading
new_name,new_server,share 3
as one line, it reads it as 2 lines
new_name,new_server,share
3
I have tried changing the value of IFS to
IFS=$'\n' #and
IFS='
'
however neither work. How can I create an array from the above file separated by newlines. My code is below.
file_formatted=$(cat ~/location/to/file/test.txt)
IFS='
' # also tried $'\n'
drives=($(sed 's/^.*:DRIVES //; s/:.*$//' <<< $file_formatted))
for line in "${drives[#]}"
do
#seperates lines into indiviudal parts
drive="$(echo $line | cut -d, -f2)"
server="$(echo $line | cut -d, -f3)"
share="$(echo $line | cut -d, -f4 | tr '\' '/' | tr '[:upper:]' '[:lower:]')"
#mount //$server/$share using osascript
#script breaks because it tries to mount /server/share instead of /server/share 3
EDIT:
tried this and got the same output as before:
drives=$(sed 's/^.*:DRIVES //; s/:.*$//' <<< $file_formatted)
while IFS= read -r line; do
printf '%s\n' "$line"
done <<< "$drives"
This is the correct way to iterate over your file; no arrays needed.
{
# Skip over lines until we read :DRIVES
while IFS= read -r line; do
[[ $line = :DRIVES ]] && break
done
# Split each comma-separated line into the desired variables,
# until we read :NAMES, wt which point we break out of this loop
while IFS=, read -r drive server share; do
[[ $drive == :NAMES ]] && break
# Use $drive, $server, and $share
done
# No need to read the rest of the file, if any
} < ~/location/to/file/test.txt
Please find below function (keyvalue.sh) that parses a configuration file with key value pairs to return the value for passed argument key.
It works fine, if the value don't have any = (equals to operator), but if the value contains = (equals to) operator, it returns incorrect value.
function getValueForKey(){
while read -r line
do
#echo $line
key=`echo $line | cut -d = -f1`
value=`echo $line | cut -d = -f2`
if [ "$2" == "$key" ]; then
echo $value
fi;
done < "$1"
}
Please find below sample key-value configuration file (keys.txt) :-
Scala_Url="http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.tgz"
Zookeeper_Url="http://www-eu.apache.org/dist/zookeeper/stable/zookeeper-3.4.10.tar.gz"
Eclipse_Url="http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/neon/3/eclipse-jee-neon-3-win32-x86_64.zip&mirror_id=1135"
Also, find below sample execution :-
$ls
keys.txt keyvalue.sh
$
$
$
$cat keys.txt
Scala_Url="http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.tgz"
Zookeeper_Url="http://www-eu.apache.org/dist/zookeeper/stable/zookeeper-3.4.10.tar.gz"
Eclipse_Url="http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/neon/3/eclipse-jee-neon-3-win32-x86_64.zip&mirror_id=1135"
$
$
$. keyvalue.sh
$
$getValueForKey keys.txt "Scala_Url"
"http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.tgz"
$
$
$
$
$getValueForKey keys.txt "Zookeeper_Url"
"http://www-eu.apache.org/dist/zookeeper/stable/zookeeper-3.4.10.tar.gz"
$
$
$
$
$
$
$getValueForKey keys.txt "Eclipse_Url"
"http://www.eclipse.org/downloads/download.php?file
$
$
$
$
$
$cat keyvalue.sh
function getValueForKey(){
while read -r line
do
#echo $line
key=`echo $line | cut -d = -f1`
value=`echo $line | cut -d = -f2`
if [ "$2" == "$key" ]; then
echo $value
fi;
done < "$1"
}$
$
$
$
$
You shouldn't use cut at all for this:
getValueForKey(){
while IFS== read -r key value;
do
if [ "$2" = "$key" ]; then
echo "$value"
fi;
done < "$1"
}
read will split the line on the input separator =, and if there are more fields than named variables, it assigns all of the remaining line to the final variable named (in this case, value).
But really you should change your format. At the very least, sort the input and use look to find the values.
William Pursell's helpful answer is an effective pure shell solution, but such solutions are inevitably slow, which is why William recommends a key-sorted configuration file combined with look.
An alternative that doesn't require sorting is to use awk:
getValueForKey() {
awk -F= -v key="$2" '$1 == key { sub(/^[^=]+=/, ""); print }' "$1"
}
-F= splits each line into fields by all occurrences of =
We can still use $1, the 1st field (key field), to compare it to the key value of interest.
To output the corresponding value, however, a string substitution is used to ensure that the value is output as-is, even if it contains = instances:
sub(/^[^=]+=/, "") replaces (sub()) everything from the start of the line (^) up to the first = instance ([^=]+ matches a nonempty sequence (+) of characters other than (^) = followed by =) with the empty string, leaving just the value.
Sample call:
$ getValueForKey keys.txt 'Eclipse_Url'
"http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/neon/3/eclipse-jee-neon-3-win32-x86_64.zip&mirror_id=1135"
I have a file of filenames, and I need to be able to get the size of these files using bash.
I have the following script which does that, but It prints the filename and the size on different lines, i'd prefer it to do it all on one line if possible.
#!/bin/sh
filename="$1"
while read -r line
do
name=$line
vars=(`echo $name | tr '.' ' '`)
echo $name
stat -x $name | grep Size: | awk '{ print $2 }'
done < "$filename"
I'd love to have it of the form:
filename: $size
How can I do this?
(I am using OSX hence the slightly odd version of stat.)
Pass -n to the echo to prevent a trailing newline from being added. So change
echo $name
to
echo -n $name
and to add the : separator between the file name and file size
echo -n ${name}": "
This should do the trick:
while read f
do
echo "${f} : $(stat -L -c %s ${f})"
done < "${filename}"
echo $name: $(stat -x $name | sed -n '/^Size:/s///p')