I'd like to parse a Terraform file named versions.tf which give provider requirements for the project. This should be in bash script.
This file is formatted like this :
terraform {
required_providers {
archive = {
source = "hashicorp/archive"
version = "~> 1.3.0"
}
aws = {
source = "hashicorp/aws"
version = "~> 3.0.0"
}
local = {
source = "hashicorp/local"
version = "~> 2.0.0"
}
template = {
source = "hashicorp/template"
version = "~> 2.2.0"
}
random = {
source = "hashicorp/random"
version = "~> 3.0.1"
}
}
required_version = ">= 0.13"
}
The goal is to have 3 variables likes so in a for loop :
$name = "archive"
$source = "hashicorp/archive"
$version = "1.3.0" # we don't take care of the version constraints
The structure is almost the same for different projects we have.
I've already tried to parse it, but as I'm a noob in text parsing, nothing concrete.
I also tried to use the command terraform providers schema -json but it doesn't work while you don't have initialized the terraform script (it need to be not initialized).
As I will use the script in my corporate Jenkins pipeline, I can't access Internet and I don't have access to binaries like jq or something not "red hat" native.
Thanks you for your help !
This script scans for the lines with the name, which are identifiable by the third field being {, and after each of these reads the two lines with source and version, assigning to variables with the same name:
while read name _ rest
do if [ "$rest" = "{" ]
then read var _ val; eval $var=$val
read var _ val; eval $var=$val
echo $name $source ${version#~> }
fi
done <versions.tf
Might do what you wanted.
#!/usr/bin/env bash
while read -r line; do
[[ $line != *'='* ]] && continue
if [[ $line == *'= {' ]]; then
line=${line%=*\{}
printf '$name = "%s"\n' "${line% }"
else
printf '$%s\n' "${line//~> }"
fi
done < versions.tf
If the required_version line needs to be omitted add
[[ $line == 'required_version'* ]] && continue
Below the line that has continue
Related
I am trying to create a unix shell/bash function to get a specific value from a YAML file.
I don't want to use Python/Ruby or other scripting languages or do not want to install any packages that are not natively available in bash.
The function should take the property name name and yaml file name as input and return the value of the property.
Sample YAML is given below.
---
Action: start
Version: 642
Domains:
Domain:
Name: SanityTest
AppSpaces:
AppSpace:
Name: SanityAppSpace
AppNodes:
AppNode:
Name: SanityAppnode
Applications:
Application:
Name: InstagramTest.application
Version: "1.0"
AppNode:
Name: SanityAppnode_1
Applications:
Application:
Name: InstagramTest.application
Version: "1.0"
I am looking for some function that works like:
DomainName=getValue Domains_Domain[1]_Name /path/my_yaml.yml
AppSpaceName=getValue Domains_Domain[$DomainName]_AppSpaces_AppSpace[1]_Name /path/my_yaml.yml
AppNodeName=getValue Domains_Domain[$DomainName]_AppSpaces_AppSpace[$AppSpaceName]_AppNodes_AppNode[1]_Name /path/my_yaml.yml
AppName=getValue Domains_Domain[$DomainName]_AppSpaces_AppSpace[$AppSpaceName]_AppNodes_AppNode[$AppNodeName]_Applications_Application[1]_Name /path/my_yaml.yml
AppVersion=getValue Domains_Domain[$DomainName]_AppSpaces_AppSpace[$AppSpaceName]_AppNodes_AppNode[$AppNodeName]_Applications_Application[$AppName]_Version /path/my_yaml.yml
I came up with the below code so far, but it is not working as expected. Appreciate any help. Please advise if my approach is wrong and if there is better way.
function getValue {
local ymlFile=$2
local Property=$1
IFS='_' read -r -a props_tokens <<< "${Property}"
local props_index=0
echo "${props_tokens[props_index]}"
CheckingFor="Domains"
currEntity_Index=0
while IFS= read -r line; do
curr_prop="${props_tokens[props_index]}"
curr_prop_Index=0
IFS=':' read -r -a curr_line_props <<< "${line}"
curr_line_prop = ${curr_line_props[0]}
if [ "${props_tokens[props_index]}" == *"["*"]"* ]
then
IFS='[' read -r -a prop_tokens <<< "${props_tokens[props_index]}"
curr_prop=${prop_tokens[0]}
curr_prop_Index=${prop_tokens[1]::-1}
echo "Index processed"
echo $curr_prop
echo $curr_prop_Index
else echo "No Index for this property"
fi
if [ "$curr_line_prop" != "$CheckingFor" ]
then continue
fi
if [ "|Action|Name|Version|" == *"$curr_prop"* ]
then
return curr_line_props[1]
fi
if [ "$curr_prop" == "$CheckingFor" ]
then
if [ "|Domains|AppSpaces|AppNodes|Applications|" == *"$curr_prop"* ]
then
props_index++
case $CheckingFor in
Domains)
CheckingFor="Domain";;
Domain)
CheckingFor="AppSpaces";;
AppSpaces)
CheckingFor="AppSpace";;
AppSpace)
CheckingFor="AppNodes";;
AppNodes)
CheckingFor="AppNode";;
AppNode)
CheckingFor="Applications";;
Applications)
CheckingFor="Application";;
*) echo "$curr_prop - not switched";;
esac
continue
else
if [ "$curr_prop_Index" == 0 ]
then
case $CheckingFor in
Domains)
CheckingFor="Domain";;
Domain)
CheckingFor="AppSpaces";;
AppSpaces)
CheckingFor="AppSpace";;
AppSpace)
CheckingFor="AppNodes";;
AppNodes)
CheckingFor="AppNode";;
AppNode)
CheckingFor="Applications";;
Applications)
CheckingFor="Application";;
*) echo "$curr_prop - not switched";;
esac
props_index++
continue
else
Integer_regex='^[0-9]+$'
if [[ $curr_prop_Index =~ $Integer_regex ]]
then
# Iterate until we get to the nth Item
currEntity_Index++
if [ $currEntity_Index == $curr_prop_Index ]
then
case $CheckingFor in
Domains)
CheckingFor="Domain";;
Domain)
CheckingFor="AppSpaces";;
AppSpaces)
CheckingFor="AppSpace";;
AppSpace)
CheckingFor="AppNodes";;
AppNodes)
CheckingFor="AppNode";;
AppNode)
CheckingFor="Applications";;
Applications)
CheckingFor="Application";;
*) echo "$curr_prop - not switched";;
esac
currEntity_Index=0
else
fi
props_index++
continue
else
# Iterate until the name of the entity match and continue with next property token
# How to handle if the search sipllied into next object?
fi
fi
fi
else
props_index++
continue
fi
done < $ymlFile
return "${props_tokens[props_index]}"
}
I have a bash script that runs, and I'm trying to use a Perl one-liner to replace some text in a file variables.php
However, I would like to check if the Perl one-liner runs successfully and that's where I get hung up. I could just output the one-liner and it would work fine, but I would like to know for sure that it ran.
Basically, the function replace_variables() is the function that does the update, and it's the if statement there that I would like to check if my one-liner worked properly.
I've tried using the run_command function in that if statement, but that did not work, and I've tried putting the one-liner directly there, which also didn't work.
If I don't wrap it in an if statement, and just call the one-liner directly, everything works as intended.
here's the full file
#!/bin/bash
export CLI_CWD="$PWD"
site_variables() {
if [ -f "$CLI_CWD/variables.php" ]; then
return true
else
return false
fi
}
replace_variables() {
# perl -pi -e 's/(dbuser)(\s+)=\s.*;$/\1 = Config::get("db")["user"];/; s/(dbpass)(\s+)=\s.*;$/\1 = Config::get("db")["pass"];/; s/(dbname)(\s+)=\s.*;$/\1 = Config::get("db")["database"];/' "$CLI_CWD/variables.php"
if [run_command ]; then
echo "Updated variables.php successfully"
else
echo "Did not update variables.php"
fi
}
run_command() {
perl -pi -e 's/(dbuser)(\s+)=\s.*;$/\1 = Config::get("db")["user"];/; s/(dbpass)(\s+)=\s.*;$/\1 = Config::get("db")["pass"];/; s/(dbname)(\s+)=\s.*;$/\1 = Config::get("db")["database"];/' "$CLI_CWD/variables.php"
}
if [ site_variables ]; then
replace_variables
else
>&2 echo "Current directory ($(pwd)) is not a project root directory"
exit 4
fi
here's the function where the if statement fails
replace_variables() {
# perl -pi -e 's/(dbuser)(\s+)=\s.*;$/\1 = Config::get("db")["user"];/; s/(dbpass)(\s+)=\s.*;$/\1 = Config::get("db")["pass"];/; s/(dbname)(\s+)=\s.*;$/\1 = Config::get("db")["database"];/' "$CLI_CWD/variables.php"
if [run_command ]; then
echo "Updated variables.php successfully"
else
echo "Did not update variables.php"
fi
}
You can see that I commented out the one-liner just before the if statement, it works if I let that run and remove the if/else check.
here is the original file snippet before the update
//Load from Settings DB
$dbuser = 'username';
$dbpass = 'password';
$dbname = 'database_name';
here is the file snippet after the update would run
//Load from Settings DB
$dbuser = Config::get("db")["user"];
$dbpass = Config::get("db")["pass"];
$dbname = Config::get("db")["database"];
tl;dr and Solution
This usage of if with [ ] will not give you the result you expect.
What you're looking for
...
if run_command; then
...
Longer explanation
Basics of if
if is a shell feature
based on the condition, it executes the body contained in between then and fi
the "condition" that if checks is a command
commands usually have a return/exit code. typically
0 for success
1 (common) and everything else for some error
e.g. 127 for command not found
when the return/exit code is 0, the body is executed
otherwise it is skipped; or control is passed to elif or else
the syntax is if <command>; then...
Where does that [ ] come from?
test is a command that can check file types and compare values
refer man test and help test (bash only)
[ ... ] is a synonym for test
NB the brackets should be surrounded by spaces on both sides
if [ -f "/path/to/$filename" ]; then
exception: when terminated by new line or ; space not required
test (or [ ]) evaluates expressions and cannot execute other commands or functions
if [ expr ]; then is alternate syntax for if test expr; then
PS: good practice to "quote" your "$variables" when used with test or [ ]
PPS: [[ ... ]] is a different thing altogether. not POSIX; available only in some shells. take a look at this thread on the UNIX Stack Exchange
I've created a simple pipeline which is attempting to run a script and then I'll do something else with the output, however the script (CheckTagsDates.sh) never finishes according to Jenkins. If I SSH into the Jenkins slave node, su as the jenkins user, navigate to the correct workspace folder, I can execute the command successfully.
pipeline {
agent {label 'agent'}
stages {
stage('Check for releases in past 24hr') {
steps{
sh 'chmod +x CheckTagsDates.sh'
script {
def CheckTagsDates = sh(script: './CheckTagsDates.sh', returnStdout: true)
echo "${CheckTagsDates}"
}
}
}
}
}
Here is the contents of the CheckTagsDates.sh file
#!/bin/bash
while read line
do
array[ $i ]="$line"
(( i++ ))
done < <( curl -L -s 'https://registry.hub.docker.com/v2/repositories/library/centos/tags'|jq -r '."results"[] | "\(.name)&\(.last_updated)"')
for i in "${array[#]}"
do
echo $i | cut -d '&' -f 1
echo $i | cut -d '&' -f 2
done
Here is the output from the script in the console
latest
2020-01-18T00:42:35.531397Z
centos8.1.1911
2020-01-18T00:42:33.410905Z
centos8
2020-01-18T00:42:29.783497Z
8.1.1911
2020-01-18T00:42:19.111164Z
8
2020-01-18T00:42:16.802842Z
centos7.7.1908
2019-11-12T00:42:46.131268Z
centos7
2019-11-12T00:42:41.619579Z
7.7.1908
2019-11-12T00:42:34.744446Z
7
2019-11-12T00:42:24.00689Z
centos7.6.1810
2019-07-02T14:42:37.943412Z
How I told you in a comment, I think that is a wrong use of the echo instruction for string interpolation.
Jenkins Pipeline uses rules identical to Groovy for string interpolation. Groovy’s String interpolation support can be confusing to many newcomers to the language. While Groovy supports declaring a string with either single quotes, or double quotes, for example:
def singlyQuoted = 'Hello'
def doublyQuoted = "World"
Only the latter string will support the dollar-sign ($) based string interpolation, for example:
def username = 'Jenkins'
echo 'Hello Mr. ${username}'
echo "I said, Hello Mr. ${username}"
Would result in:
Hello Mr. ${username}
I said, Hello Mr. Jenkins
Understanding how to use string interpolation is vital for using some of Pipeline’s more advanced features.
Source: https://jenkins.io/doc/book/pipeline/jenkinsfile/#string-interpolation
As a workaround for this case, I would suggest you to do the parsing of the json content in Groovy, instead of shell, and limit the script to only retrieving the json.
pipeline {
agent {label 'agent'}
stages {
stage('Check for releases in past 24hr') {
steps{
script {
def TagsDates = sh(script: "curl -L -s 'https://registry.hub.docker.com/v2/repositories/library/centos/tags'", returnStdout: true).trim()
TagsDates = readJSON(text: TagsDates)
TagsDates.result.each {
echo("${it.name}")
echo("${it.last_updated}")
}
}
}
}
}
}
I'm trying to do ASR system. Im using kaldi manual and librispeech corpus.
In data preparation step i get this error
utils/data/get_utt2dur.sh: segments file does not exist so getting durations
from wave files
utils/data/get_utt2dur.sh: could not get utterance lengths from sphere-file
headers, using wav-to-duration
utils/data/get_utt2dur.sh: line 99: wav-to-duration: command not found
And here the piece of code where this error occures
if cat $data/wav.scp | perl -e '
while (<>) { s/\|\s*$/ |/; # make sure final | is preceded by space.
#A = split;
if (!($#A == 5 && $A[1] =~ m/sph2pipe$/ &&
$A[2] eq "-f" && $A[3] eq "wav" && $A[5] eq "|")) { exit (1); }
$utt = $A[0]; $sphere_file = $A[4];
if (!open(F, "<$sphere_file")) { die "Error opening sphere file $sphere_file"; }
$sample_rate = -1; $sample_count = -1;
for ($n = 0; $n <= 30; $n++) {
$line = <F>;
if ($line =~ m/sample_rate -i (\d+)/) { $sample_rate = $1; }
if ($line =~ m/sample_count -i (\d+)/) { $sample_count = $1;
}
if ($line =~ m/end_head/) { break; }
}
close(F);
if ($sample_rate == -1 || $sample_count == -1) {
die "could not parse sphere header from $sphere_file";
}
$duration = $sample_count * 1.0 / $sample_rate;
print "$utt $duration\n";
} ' > $data/utt2dur; then
echo "$0: successfully obtained utterance lengths from sphere-file headers"
else
echo "$0: could not get utterance lengths from sphere-file headers,
using wav-to-duration"
if command -v wav-to-duration >/dev/null; then
echo "$0: wav-to-duration is not on your path"
exit 1;
fi
In file wav.scp i got such lines:
6295-64301-0002 flac -c -d -s /home/tinin/kaldi/egs/librispeech/s5/LibriSpeech/dev-clean/6295/64301/6295-64301-0002.flac |
In this dataset i have only flac files(they downloaded via provided script) and i dont understand why we search wav-files? And how run data preparation correctly(i didnt change source code in this manual.
Also, if you explain to me what is happening in this code, then I will be very grateful to you, because i'm not familiar with bash and perl.
Thank you a lot!
The problem I see from this line
utils/data/get_utt2dur.sh: line 99: wav-to-duration: command not found
is that you have not added the kaldi tools in your path.
Check the file path.sh and see if the directories that it adds to your path are correct (because it has ../../.. inside and it might not match your current folder setup)
As for the perl script, it counts the samples of the sound file and then it divides with the sample rate in order to get the duration. Don't worry about the 'wav' word, your files might be on another format, it's just the name of the kaldi functions.
See the discussion at Is `command -v` option required in a POSIX shell? Is posh compliant with POSIX?. It describes that type as well as command -v option is optional in POSIX.1-2004.
The answer marked correct at Check if a program exists from a Bash script doesn't help either. Just like type, hash is also marked as XSI in POSIX.1-2004. See http://pubs.opengroup.org/onlinepubs/009695399/utilities/hash.html.
Then what would be a POSIX compliant way to write a shell script to find if a command exists on the system or not?
How do you want to go about it? You can look for the command on directories in the current value of $PATH; you could look in the directories specified by default for the system PATH (getconf PATH as long as getconf
exists on PATH).
Which implementation language are you going to use? (For example: I have a Perl implementation that does a decent job finding executables on $PATH — but Perl is not part of POSIX; is it remotely relevant to you?)
Why not simply try running it? If you're going to deal with Busybox-based systems, lots of the executables can't be found by searching — they're built into the shell. The major caveat is if a command does something dangerous when run with no arguments — but very few POSIX commands, if any, do that. You might also need to determine what command exit statuses indicate that the command is not found versus the command objecting to not being called with appropriate arguments. And there's little guarantee that all systems will be consistent on that. It's a fraught process, in case you hadn't gathered.
Perl implementation pathfile
#!/usr/bin/env perl
#
# #(#)$Id: pathfile.pl,v 3.4 2015/10/16 19:39:23 jleffler Exp $
#
# Which command is executed
# Loosely based on 'which' from Kernighan & Pike "The UNIX Programming Environment"
#use v5.10.0; # Uses // defined-or operator; not in Perl 5.8.x
use strict;
use warnings;
use Getopt::Std;
use Cwd 'realpath';
use File::Basename;
my $arg0 = basename($0, '.pl');
my $usestr = "Usage: $arg0 [-AafhqrsVwx] [-p path] command ...\n";
my $hlpstr = <<EOS;
-A Absolute pathname (determined by realpath)
-a Print all possible matches
-f Print names of files (as opposed to symlinks, directories, etc)
-h Print this help message and exit
-q Quiet mode (don't print messages about files not found)
-r Print names of files that are readable
-s Print names of files that are not empty
-V Print version information and exit
-w Print names of files that are writable
-x Print names of files that are executable
-p path Use PATH
EOS
sub usage
{
print STDERR $usestr;
exit 1;
}
sub help
{
print $usestr;
print $hlpstr;
exit 0;
}
sub version
{
my $version = 'PATHFILE Version $Revision: 3.4 $ ($Date: 2015/10/16 19:39:23 $)';
# Beware of RCS hacking at RCS keywords!
# Convert date field to ISO 8601 (ISO 9075) notation
$version =~ s%\$(Date:) (\d\d\d\d)/(\d\d)/(\d\d) (\d\d:\d\d:\d\d) \$%\$$1 $2-$3-$4 $5 \$%go;
# Remove keywords
$version =~ s/\$([A-Z][a-z]+|RCSfile): ([^\$]+) \$/$2/go;
print "$version\n";
exit 0;
}
my %opts;
usage unless getopts('AafhqrsVwxp:', \%opts);
version if ($opts{V});
help if ($opts{h});
usage unless scalar(#ARGV);
# Establish test and generate test subroutine.
my $chk = 0;
my $test = "-x";
my $optlist = "";
foreach my $opt ('f', 'r', 's', 'w', 'x')
{
if ($opts{$opt})
{
$chk++;
$test = "-$opt";
$optlist .= " -$opt";
}
}
if ($chk > 1)
{
$optlist =~ s/^ //;
$optlist =~ s/ /, /g;
print STDERR "$arg0: mutually exclusive arguments ($optlist) given\n";
usage;
}
my $chk_ref = eval "sub { my(\$cmd) = \#_; return -f \$cmd && $test \$cmd; }";
my #PATHDIRS;
my %pathdirs;
my $path = defined($opts{p}) ? $opts{p} : $ENV{PATH};
#foreach my $element (split /:/, $opts{p} // $ENV{PATH})
foreach my $element (split /:/, $path)
{
$element = "." if $element eq "";
push #PATHDIRS, $element if $pathdirs{$element}++ == 0;
}
my $estat = 0;
CMD:
foreach my $cmd (#ARGV)
{
if ($cmd =~ m%/%)
{
if (&$chk_ref($cmd))
{
print "$cmd\n" unless $opts{q};
next CMD;
}
print STDERR "$arg0: $cmd: not found\n" unless $opts{q};
$estat = 1;
}
else
{
my $found = 0;
foreach my $directory (#PATHDIRS)
{
my $file = "$directory/$cmd";
if (&$chk_ref($file))
{
$file = realpath($file) if $opts{A};
print "$file\n" unless $opts{q};
next CMD unless defined($opts{a});
$found = 1;
}
}
print STDERR "$arg0: $cmd: not found\n" unless $found || $opts{q};
$estat = 1;
}
}
exit $estat;