Validate log file using shell - shell

Question

I want to validate a log file based on a reference file, I worked on a script but, it is not beautiful and is not optimal:
For each line I want to check the value of the fields,
- the field 7 equal to 1 I have to check columns 16 and 17
- the field 7 equal to 2 I have to check columns 25 and 27 and 30
- the field 7 equal to 3 I have to check columns 18 and 24 and 31
etc..
#!/bin/bash
LOG=SMS.log
awk -F\| ' {s=""}
$4!=0 {printf "API has wrong value"; s="; " }
$8=="" { printf "%sApplicationID is empty", s; s="; " }
$9=="" { printf "%shttp request method is empty", s; s="; " }
$7=="" { printf "%sOperationID is empty", s; s="; " }
$13 !~ /0|1|2/ {printf "%sresult(0,1,2) has a wrong value", s; s="; " }
# 1:create SMS
$7=="1" && $18=="" {printf "%sSender is missing", s; s="; " }
$7=="1" && $18 ~ /\/tel\:\+\*\*/ {printf "%sSender is cyphred !", s; s="; " }
$7=="1" && $20=="" {printf "%sAddress is missing", s; s="; " }
$7=="1" && $20 ~ /\/tel\:\+[0-9]/ {printf "%sAddress(es) is not cyphred", s; s="; " }
$7=="1" && $10 ~ /\/tel\:\+\*\*/ {printf "%sSender is cyphred on URI !", s; s="; " }
## 2:subscribe
$7=="2" && $25=="" {printf "%sdestination is missing", s; s="; " }
$7=="2" && $16=="201" && $27="" {printf "%sresourceId is missing", s; s="; "}
#3:unsubscribe
$7=="2" && $16=="201" && $25="" {printf "%sresource is missing", s; s="; "}
s { printf "\n"}
s
{printf "\n"}
' $LOG
Is it possible to update the code to be more optimal and beautiful.
Output:
Application is empty; Operation is empty; Http request method is empty
83ac|EDR|V1|0|V1|2019-05-14|7||||2019-05-14T08:00:42.758Z|8|0|||||XXXXX|||||||||789|||||||||5945548f|||||

Answer 1

I'd do it like this:
awk -F'|' '
##### Error Detection
$4 != 0 { prtErr("ApiWrong") }
$8 == "" { prtErr("AppIdEmpty") }
$9 == "" { prtErr("HttpEmpty") }
$7 == "" { prtErr("OpIdEmpty") }
$13 !~ /[012]/ { prtErr("RsltBad") }
$7 == 1 { # 1:create SMS
if ( $18 == "" ) { prtErr("SndMiss") }
if ( $18 ~ /\/tel:\+\*\*/ ) { prtErr("SndCyph") }
if ( $20 == "" ) { prtErr("AddrMiss") }
if ( $20 ~ /\/tel:\+[0-9]/ ) { prtErr("AddrNotCyph") }
if ( $10 ~ /\/tel:\+\*\*/ ) { prtErr("SndCyphUri") }
}
$7 == 2 { # 2:subscribe
if ( $25 == "" ) { prtErr("DestMiss") }
if ( $16=="201" && $27=="" ) { prtErr("RsrcIdMiss") }
}
$7 == 3 { # 3:unsubscribe
if ( $16=="201" && $25=="" ) { prtErr("RsrcMiss") }
}
##### Error Reporting
function prtDbg(code,str) { if (doDebug) prtMsg("DEBUG",code,str) }
function prtTrc(code,str) { if (doTrace) prtMsg("TRACE",code,str) }
function prtWrn(code,str) { prtMsg("WARNING",code,str) }
function prtErr(code,str) { prtMsg("ERROR",code,str) }
function prtMsg(level, code, str, map, msg) {
map["ApiWrong"] = "API has wrong value"
map["AppIdEmpty"] = "ApplicationID is empty"
map["HttpEmpty"] = "http request method is empty"
map["OpIdEmpty"] = "OperationID is empty"
map["RsltBad"] = "result(0,1,2) has a wrong value"
map["SndMiss"] = "Sender is missing"
map["SndCyph"] = "Sender is cyphred !"
map["AddrMiss"] = "Address is missing"
map["AddrNotCyph" = "Address(es) is not cyphred"
map["SndCyphUri"] = "Sender is cyphred on URI !"
map["DestMiss"] = "destination is missing"
map["RsrcIdMiss"] = "resourceId is missing"
map["RsrcMiss"] = "resource is missing"
map["default"] = "Unknown error code"
msg = (code in map ? map[code] : map["default"])
printf "%s: %s[%d]: (%s) %s\n", level, FILENAME, FNR, code, msg | "cat>&2"
if ( str != "" ) {
printf "%s: %s[%d]:\t%s\n", $0 | "cat>&2"
}
}
' "$log"
That decouples the text being printed from the error indication and centralizes/instruments all error messages for a common look/feel and ability to add extra info if necessary and to de-clutter the code that's detecting the errors. I also showed how to separate errors, from warnings, etc. (you choose which is which in your code) and add tracing/debugging functions that you can all over the code if you like but to and they won't do anything till you set the relevant "do..." flag on the command line.
Update to just produce the specific output you asked for (untested):
BEGIN { FS="|" }
##### General processing including error detection
$4 != 0 { logErr("ApiWrong") }
$8 == "" { logErr("AppIdEmpty") }
$9 == "" { logErr("HttpEmpty") }
$7 == "" { logErr("OpIdEmpty") }
$13 !~ /[012]/ { logErr("RsltBad") }
$7 == 1 { # 1:create SMS
if ( $18 == "" ) { logErr("SndMiss") }
if ( $18 ~ /\/tel:\+\*\*/ ) { logErr("SndCyph") }
if ( $20 == "" ) { logErr("AddrMiss") }
if ( $20 ~ /\/tel:\+[0-9]/ ) { logErr("AddrNotCyph") }
if ( $10 ~ /\/tel:\+\*\*/ ) { logErr("SndCyphUri") }
}
$7 == 2 { # 2:subscribe
if ( $25 == "" ) { logErr("DestMiss") }
if ( $16=="201" && $27=="" ) { logErr("RsrcIdMiss") }
}
$7 == 3 { # 3:unsubscribe
if ( $16=="201" && $25=="" ) { logErr("RsrcMiss") }
}
{ prtErrs() }
##### Error reporting primitives
function logErr(code) { _errs[code] }
function prtErrs( code, map, msg, gotErrs, sep) {
for (code in _errs) {
gotErrs = 1
break
}
if (gotErrs) {
map["ApiWrong"] = "API has wrong value"
map["AppIdEmpty"] = "ApplicationID is empty"
map["HttpEmpty"] = "http request method is empty"
map["OpIdEmpty"] = "OperationID is empty"
map["RsltBad"] = "result(0,1,2) has a wrong value"
map["SndMiss"] = "Sender is missing"
map["SndCyph"] = "Sender is cyphred !"
map["AddrMiss"] = "Address is missing"
map["AddrNotCyph"] = "Address(es) is not cyphred"
map["SndCyphUri"] = "Sender is cyphred on URI !"
map["DestMiss"] = "destination is missing"
map["RsrcIdMiss"] = "resourceId is missing"
map["RsrcMiss"] = "resource is missing"
printf "%s: %s[%d]: ", "ERROR", FILENAME, FNR | "cat>&2"
for (code in _errs) {
msg = (code in map ? map[code] : "Unknown error code (" code ")")
printf "%s%s", sep, msg | "cat>&2"
sep = "; "
}
printf "\n%s\n", $0 | "cat>&2"
delete _errs
}
}
and if you have GNU awk for arrays of arrays and length(array) then I'd do it as:
BEGIN { FS="|" }
##### General processing including error detection
$4 != 0 { logErr("Wrong","API") }
$8 == "" { logErr("Empty","AppId") }
$9 == "" { logErr("Empty","Http request method") }
$7 == "" { logErr("Empty","OperationID") }
$13 !~ /[012]/ { logErr("Wrong","Result(0,1,2)") }
$7 == 1 { # 1:create SMS
if ( $18 == "" ) { logErr("Miss","Sender") }
if ( $18 ~ /\/tel:\+\*\*/ ) { logErr("Cyph","Sender") }
if ( $20 == "" ) { logErr("Miss","Address") }
if ( $20 ~ /\/tel:\+[0-9]/ ) { logErr("NotCyph","Address(es)") }
if ( $10 ~ /\/tel:\+\*\*/ ) { logErr("UriCyph","Sender") }
}
$7 == 2 { # 2:subscribe
if ( $25 == "" ) { logErr("Miss","Destination") }
if ( $16=="201" && $27=="" ) { logErr("Miss","ResourceId") }
}
$7 == 3 { # 3:unsubscribe
if ( $16=="201" && $25=="" ) { logErr("Miss","Resource") }
}
{ prtErrs() }
##### Error reporting primitives
function logErr(type,item) { _errs[type][item] }
function prtErrs( map, type, msg, item, sep) {
if ( length(_errs) ) {
map["Wrong"] = "has wrong value"
map["Empty"] = "is empty"
map["Miss"] = "is missing"
map["Cyph"] = "is cyphred !"
map["NotCyph"] = "is not cyphred"
map["UriCyph"] = "is cyphred on URI !"
printf "%s: %s[%d]: ", "ERROR", FILENAME, FNR | "cat>&2"
for (type in _errs) {
msg = (type in map ? map[type] : "Unknown error type (" type ")")
for (item in _errs[type]) {
printf "%s%s %s", sep, item, msg | "cat>&2"
sep = "; "
}
}
printf "\n%s\n", $0 | "cat>&2"
delete _errs
}
}

Answer 2

First thing you could do is get rid of the s variable.
#!/bin/bash
LOG=SMS.log
awk -F\| '
function add_error(message){
error = error OFS message
}
$4!=0 {add_error("API has wrong value")}
$8=="" {add_error("ApplicationID is empty")}
$9=="" {add_error("http request method is empty")}
$7=="" {add_error("OperationID is empty")}
$13 !~ /0|1|2/ {add_error("result(0,1,2) has a wrong value")}
# 1:create SMS
$7=="1" && $18=="" {add_error("Sender is missing")}
$7=="1" && $18 ~ /\/tel\:\+\*\*/ {add_error("Sender is cyphred !")}
$7=="1" && $20=="" {add_error("Address is missing")}
$7=="1" && $20 ~ /\/tel\:\+[0-9]/ {add_error("Address(es) is not cyphred")}
$7=="1" && $10 ~ /\/tel\:\+\*\*/ {add_error("Sender is cyphred on URI !")}
## 2:subscribe
$7=="2" && $25=="" {add_error("destination is missing")}
$7=="2" && $16=="201" && $27="" {add_error("resourceId is missing")}
#3:unsubscribe
$7=="2" && $16=="201" && $25="" {add_error("resource is missing")}
{
print substr(error, length(OFS)+1); #Works even if error is empty
error = "";
}
' OFS="; " $LOG
I think that it is a bit strange to analyze your log file and create ... a new log file. Why don't you create a csv with 1 column per error and 1/0 values for each line/error ? Your result would be much more easier to analyze and would contains all the informations you need.