Related
I have a raw audio file that I would like to transcript using watson speech to text service. I am using the default example provided at the repo.
But it works if I use http interface.
// working code
package main
import (
"os"
"github.com/IBM/go-sdk-core/v5/core"
"github.com/watson-developer-cloud/go-sdk/v2/speechtotextv1"
)
func main() {
// Instantiate the Watson Speech To Text service
authenticator := &core.IamAuthenticator{
ApiKey: "YOUR API KEY",
}
service, serviceErr := speechtotextv1.
NewSpeechToTextV1(&speechtotextv1.SpeechToTextV1Options{
URL: "YOUR SERVICE URL",
Authenticator: authenticator,
})
// Check successful instantiation
if serviceErr != nil {
panic(serviceErr)
}
// Open file with mp3 to recognize
audio, audioErr := os.Open("/opt/audioRaw.raw")
if audioErr != nil {
panic(audioErr)
}
// Create a new RecognizeOptions for ContentType "audio/mp3"
recognizeOptions := service.
NewRecognizeOptions(audio).
SetContentType("audio/mulaw;rate=8000;channels=1").SetModel("en-US_NarrowbandModel")
// Call the speechToText Recognize method
recognizeResult, _, responseErr := service.Recognize(recognizeOptions)
// Check successful call
if responseErr != nil {
panic(responseErr)
}
// Check successful casting
if recognizeResult != nil {
core.PrettyPrint(recognizeResult, "Recognize")
}
}
But if i use websocket for the raw audio file, it does not work and panic with below error upon debugging.
"error": "unable to transcode data stream application/octet-stream -> audio/l16 "
// Does not work
package main
import (
"encoding/json"
"fmt"
"os"
"github.com/IBM/go-sdk-core/v5/core"
"github.com/watson-developer-cloud/go-sdk/v2/speechtotextv1"
)
func main() {
// Instantiate the Watson Speech To Text service
authenticator := &core.IamAuthenticator{
ApiKey: "YOUR API KEY",
}
service, serviceErr := speechtotextv1.
NewSpeechToTextV1(&speechtotextv1.SpeechToTextV1Options{
URL: "YOUR SERVICE URL",
Authenticator: authenticator,
})
// Check successful instantiation
if serviceErr != nil {
panic(serviceErr)
}
// Open file with mp3 to recognize
audio, audioErr := os.Open("/opt/audioRaw.raw")
if audioErr != nil {
panic(audioErr)
}
// callbook can have `OnOpen`, `onData`, `OnClose` and `onError` functions
callback := myCallBack{}
recognizeUsingWebsocketOptions := service.
NewRecognizeUsingWebsocketOptions(audio, "audio/mulaw;rate=8000;channels=1")
recognizeUsingWebsocketOptions.
SetModel("en-US_NarrowbandModel").
SetWordConfidence(true).
SetSpeakerLabels(true).
SetTimestamps(true)
service.RecognizeUsingWebsocket(recognizeUsingWebsocketOptions, callback)
}
type myCallBack struct{}
func (cb myCallBack) OnOpen() {
fmt.Println("Handshake successful")
}
func (cb myCallBack) OnClose() {
fmt.Println("Closing connection")
}
func (cb myCallBack) OnData(resp *core.DetailedResponse) {
var speechResults speechtotextv1.SpeechRecognitionResults
result := resp.GetResult().([]byte)
json.Unmarshal(result, &speechResults)
core.PrettyPrint(speechResults, "Recognized audio: ")
}
func (cb myCallBack) OnError(err error) {
panic(err)
}
Could someone please help me in figure out why raw audio does not work through websocket interface?
Any pointer is much appreciated.
Ryan
I currently retrieve my organization (family) Google Calendars via the Google Calendar API, using Python.
For reference, the credentials file used in both cases is ((...) is redacted information)
{
"type": "service_account",
"project_id": "da(...)",
"private_key_id": "8(...)4",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAI(...)Kag==\n-----END PRIVATE KEY-----\n",
"client_email": "da(...)#(...).iam.gserviceaccount.com",
"client_id": "1(...)9",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/da(...)iam.gserviceaccount.com"
}
I would like to port this to Go. To do so, I used the following code (partly copied from the documentation)
package main
import (
"context"
"fmt"
"time"
log "github.com/sirupsen/logrus"
"google.golang.org/api/calendar/v3"
"google.golang.org/api/option"
)
func googlecalendar() (err error) {
ctx := context.Background()
calendarService, err := calendar.NewService(ctx, option.WithCredentialsFile("googlecalendar.json"))
if err != nil {
return err
}
t := time.Now().Format(time.RFC3339)
events, err := calendarService.Events.List("john#example.com").ShowDeleted(false).
SingleEvents(true).TimeMin(t).MaxResults(10).OrderBy("startTime").Do()
if err != nil {
log.Fatalf("Unable to retrieve next ten of the user's events: %v", err)
}
fmt.Println("Upcoming events:")
if len(events.Items) == 0 {
fmt.Println("No upcoming events found.")
} else {
for _, item := range events.Items {
date := item.Start.DateTime
if date == "" {
date = item.Start.Date
}
fmt.Printf("%v (%v)\n", item.Summary, date)
}
}
return nil
}
In the code above john#example.com is the "Calendar ID" provided in Google Calendar (the actual ID is used in the code of course). This outputs the error
time="2021-05-05T11:39:12+02:00" level=fatal msg="Unable to retrieve next ten of the user's events: googleapi: Error 404: Not Found, notFound"
My understanding is that this means that john#example.com was not recognized. Why?
When instead of john#example.com I use primary (as in the docs), the code runs correctly and outputs
Upcoming events:
No upcoming events found.
This means that the whole authentication part seems to work (using the same JSON credentials as in Python, retrieved from the console) - it is just that the reference to the calendar is not correct. How to fix this?
Following a request in comments, below is the general Python code (that works).
import json
import logging.config
import os
import arrow
import googleapiclient.discovery
import paho.mqtt.publish
from google.oauth2 import service_account as google_oauth2_service_account
# setup logging
logging.config.dictConfig({
'formatters': {
'standard': {
'format': "%(asctime)s [%(module)s] %(levelname)s %(message)s"
},
},
'handlers': {
'default': {
'formatter': 'standard',
'class': 'logging.StreamHandler',
},
},
'loggers': {
'': { # root logger
'handlers': ['default'],
'level': logging.ERROR,
'propagate': False
},
'googlecalendar': {
'handlers': ['default'],
'level': logging.DEBUG if os.environ.get('DEBUG', '').lower() == 'true' else logging.INFO,
'propagate': False
},
},
"disable_existing_loggers": True,
"version": 1,
})
log = logging.getLogger('googlecalendar')
def getevents(calendar=None, google_service =None):
now = arrow.now()
try:
events_result = google_service.events().list(
calendarId=calendar,
timeMin=now.shift(days=0).isoformat(),
timeMax=now.shift(days=+7).isoformat(),
singleEvents=True,
orderBy='startTime'
).execute()
except Exception as e:
log.error(f"error connecting to google for {calendar}: {e}")
return
# transform events and send out
events = []
if events_result.get('items'):
# remove multiday items (no strat or end date)
for i in events_result['items']:
# try to extract the data, some are not available so the event is not interesing (multiday, ...)
try:
events.append({
'start': arrow.get(i['start']['dateTime']).isoformat(),
'end': arrow.get(i['end']['dateTime']).isoformat(),
'timestamp': arrow.get(i['start']['dateTime']).timestamp,
'name': i['summary'],
'id': i['id'],
'important': True if i.get('colorId') else False
})
except KeyError:
pass
# send the data to MQTT
# (...)
log.debug(f"{calendar}: {events}")
google_credentials = google_oauth2_service_account.Credentials.from_service_account_info(
{
"type": "service_account",
"project_id": "da(...)",
"private_key_id": "8(...)4",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAI(...)Kag==\n-----END PRIVATE KEY-----\n",
"client_email": "da(...)#(...).iam.gserviceaccount.com",
"client_id": "1(...)9",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/da(...)iam.gserviceaccount.com"
},
scopes=['https://www.googleapis.com/auth/calendar'],
subject='john#example.com'
)
google_service = googleapiclient.discovery.build('calendar', 'v3', credentials=google_credentials, cache_discovery=False)
for calendar in [
"john#example.com",
"mary#example.net",
"something_idsd#group.calendar.google.com",
]:
getevents(calendar=calendar, google_service=google_service)
Answer
The result you are getting is normal, the problem is that you are not performing Domain-Wide Delegation correctly. I come to this conclusion because of the following scenarios:
Calendar Id: john#example.com. If the service account has not impressed the user, it is expected that the user's calendar will not be found.
Calendar Id: primary. If the primary calendar of the service account does not have any events, it is expected that the list method does not return any results.
Solution
Comparing your code with the one in the documentation, I don't see where you place config.Subject = userEmail, as #DalmTo says. Try the following code to create the calendar service:
ctx := context.Background()
jsonCredentials, err := ioutil.ReadFile("googlecalendar.json")
if err != nil {
return nil, err
}
config, err := google.JWTConfigFromJSON(jsonCredentials, "https://www.googleapis.com/auth/calendar")
if err != nil {
return nil, fmt.Errorf("JWTConfigFromJSON: %v", err)
}
config.Subject = "john#example.com"
ts := config.TokenSource(ctx)
calendarService, err := calendar.NewService(ctx, option.WithTokenSource(ts))
References:
Domain-Wide Delegation
EDIT: Adrian's suggestion makes sense, so I moved my code into a function and called the function from my cobra block:
package cmd
import (
"fmt"
"log"
"os"
"io"
"github.com/spf13/cobra"
"github.com/spf13/viper"
input "github.com/tcnksm/go-input"
)
var configureCmd = &cobra.Command{
Use: "configure",
Short: "Configure your TFE credentials",
Long: `Prompts for your TFE API credentials, then writes them to
a configuration file (defaults to ~/.tgc.yaml`,
Run: func(cmd *cobra.Command, args []string) {
CreateConfigFileFromPrompts(os.Stdin, os.Stdout)
},
}
func CreateConfigFileFromPrompts(stdin io.Reader, stdout io.Writer) {
ui := &input.UI{
Writer: stdout,
Reader: stdin,
}
tfeURL, err := ui.Ask("TFE URL:", &input.Options{
Default: "https://app.terraform.io",
Required: true,
Loop: true,
})
if err != nil {
log.Fatal(err)
}
viper.Set("tfe_url", tfeURL)
tfeAPIToken, err := ui.Ask(fmt.Sprintf("TFE API Token (Create one at %s/app/settings/tokens)", tfeURL), &input.Options{
Default: "",
Required: true,
Loop: true,
Mask: true,
MaskDefault: true,
})
if err != nil {
log.Fatal(err)
}
viper.Set("tfe_api_token", tfeAPIToken)
configPath := ConfigPath()
viper.SetConfigFile(configPath)
err = viper.WriteConfig()
if err != nil {
log.Fatal("Failed to write to: ", configPath, " Error was: ", err)
}
fmt.Println("Saved to", configPath)
}
So what can I pass to this method to test that the output is as expected?
package cmd
import (
"strings"
"testing"
)
func TestCreateConfigFileFromPrompts(t *testing.T) {
// How do I pass the stdin and out to the method?
// Then how do I test their contents?
// CreateConfigFileFromPrompts()
}
func TestCreateConfigFileFromPrompts(t *testing.T) {
var in bytes.Buffer
var gotOut, wantOut bytes.Buffer
// The reader should read to the \n each of two times.
in.Write([]byte("example-url.com\nexampletoken\n"))
// wantOut could just be []byte, but for symmetry's sake I've used another buffer
wantOut.Write([]byte("TFE URL:TFE API Token (Create one at example-url.com/app/settings/tokens)"))
// I don't know enough about Viper to manage ConfigPath()
// but it seems youll have to do it here somehow.
configFilePath := "test/file/location"
CreateConfigFileFromPrompts(&in, &gotOut)
// verify that correct prompts were sent to the writer
if !bytes.Equal(gotOut.Bytes(), wantOut.Bytes()) {
t.Errorf("Prompts = %s, want %s", gotOut.Bytes(), wantOut.Bytes())
}
// May not need/want to test viper's writing of the config file here, or at all, but if so:
var fileGot, fileWant []byte
fileWant = []byte("Correct Config file contents:\n URL:example-url.com\nTOKEN:exampletoken")
fileGot, err := ioutil.ReadFile(configFilePath)
if err != nil {
t.Errorf("Error reading config file %s", configFilePath)
}
if !bytes.Equal(fileGot, fileWant) {
t.Errorf("ConfigFile: %s not created correctly got = %s, want %s", configFilePath, fileGot, fileWant)
}
}
As highlighted by #zdebra in comments to his answer, the go-input package is panicing and giving you the error: Reader must be a file. If you are married to using that package, you can avoid the problem by disabling the masking option on the ui.Ask for your second input:
tfeAPIToken, err := ui.Ask(fmt.Sprintf("TFE API Token (Create one at %s/app/settings/tokens)", tfeURL), &input.Options{
Default: "",
Required: true,
Loop: true,
//Mask: true, // if this is set to True, the input must be a file for some reason
//MaskDefault: true,
})
The reader and the writer need to be set up before the tested function is called. After is called, the result is written into the writer where it should be verified.
package cmd
import (
"strings"
"testing"
)
func TestCreateConfigFileFromPrompts(t *testing.T) {
in := strings.NewReader("<your input>") // you can use anything that satisfies io.Reader interface here
out := new(strings.Builder) // you could use anything that satisfies io.Writer interface here like bytes.Buffer
CreateConfigFileFromPrompts(in, out)
// here you verify the output written into the out
expectedOutput := "<your expected output>"
if out.String() != expectedOutput {
t.Errorf("expected %s to be equal to %s", out.String(), expectedOutput)
}
}
I started using AWS SAM and for now I only have some unit tests, but I want to try to run integration tests in a pre traffic hook function.
Unfortunately there seems to be no code example for Golang, all I could find was for Javascript.
From this example I pieced together that I have to use the code deploy SDK and call PutLifecycleEventHookExecutionStatus, but the specifics remain unclear. The aws code example repo for go has no examples for code deploy either.
More Information about the topic that I am looking for is available here https://github.com/awslabs/serverless-application-model/blob/master/docs/safe_lambda_deployments.rst#pretraffic-posttraffic-hooks.
I want to start out by testing a lambda function that simply queries DynamoDB.
Something like this works:
package main
import (
"context"
"encoding/json"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/codedeploy"
)
type CodeDeployEvent struct {
DeploymentId string `json:"deploymentId"`
LifecycleEventHookExecutionId string `json:"lifecycleEventHookExecutionId"`
}
func HandleRequest(ctx context.Context, event CodeDeployEvent) (string, error) {
// add some tests here and change status flag as needed . . .
client := codedeploy.New(session.New())
params := &codedeploy.PutLifecycleEventHookExecutionStatusInput{
DeploymentId: &event.DeploymentId,
LifecycleEventHookExecutionId: &event.LifecycleEventHookExecutionId,
Status: "Succeeded",
}
req, _ := client.PutLifecycleEventHookExecutionStatusRequest(params)
_ = req.Send()
}
I got around to implement this and want to share my complete solution.
After figuring out how to use it, I decided against using it, because there are a couple of drawbacks.
there is no way to expose a new version of the canary to a dedicated portion of the user base, that means sometimes they'll hit the new or the old version
invoking functions that publish to sns will trigger all downstream actions, which might get the new or the old version of the downstream services, which would cause a lot of problems in case of breaking APIs
IAM changes affect both version immediately, possibly breaking the old version.
Instead, I deploy everything to a pre prod account, run my integration and e2e tests and if they succeed I'll deploy to prod
the cdk code to create a canary deployment:
const versionAlias = new lambda.Alias(this, 'Alias', {
aliasName: "alias",
version: this.lambda.currentVersion,
})
const preHook = new lambda.Function(this, 'LambdaPreHook', {
description: "pre hook",
code: lambda.Code.fromAsset('dist/upload/convert-pre-hook'),
handler: 'main',
runtime: lambda.Runtime.GO_1_X,
memorySize: 128,
timeout: cdk.Duration.minutes(1),
environment: {
FUNCTION_NAME: this.lambda.currentVersion.functionName,
},
reservedConcurrentExecutions: 5,
logRetention: RetentionDays.ONE_WEEK,
})
// this.lambda.grantInvoke(preHook) // this doesn't work, I need to grant invoke to all functions :s
preHook.addToRolePolicy(new iam.PolicyStatement({
actions: [
"lambda:InvokeFunction",
],
resources: ["*"],
effect: iam.Effect.ALLOW,
}))
const application = new codedeploy.LambdaApplication(this, 'CodeDeployApplication')
new codedeploy.LambdaDeploymentGroup(this, 'CanaryDeployment', {
application: application,
alias: versionAlias,
deploymentConfig: codedeploy.LambdaDeploymentConfig.ALL_AT_ONCE,
preHook: preHook,
autoRollback: {
failedDeployment: true,
stoppedDeployment: true,
deploymentInAlarm: false,
},
ignorePollAlarmsFailure: false,
// alarms:
// autoRollback: codedeploy.A
// postHook:
})
My go code of the pre hook function. PutLifecycleEventHookExecutionStatus tells code deploy if the pre hook succeeded or not. Unfortunately in case you fail the deployment message, the message you get in the cdk deploy output is utterly useless, so you need to check the pre/post hook logs.
In order to actually run the integration test I simply invoke the lambda and check if an error occurred.
package main
import (
"encoding/base64"
"fmt"
"log"
"os"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/codedeploy"
lambdaService "github.com/aws/aws-sdk-go/service/lambda"
)
var svc *codedeploy.CodeDeploy
var lambdaSvc *lambdaService.Lambda
type codeDeployEvent struct {
DeploymentId string `json:"deploymentId"`
LifecycleEventHookExecutionId string `json:"lifecycleEventHookExecutionId"`
}
func handler(e codeDeployEvent) error {
params := &codedeploy.PutLifecycleEventHookExecutionStatusInput{
DeploymentId: &e.DeploymentId,
LifecycleEventHookExecutionId: &e.LifecycleEventHookExecutionId,
}
err := handle()
if err != nil {
log.Println(err)
params.Status = aws.String(codedeploy.LifecycleEventStatusFailed)
} else {
params.Status = aws.String(codedeploy.LifecycleEventStatusSucceeded)
}
_, err = svc.PutLifecycleEventHookExecutionStatus(params)
if err != nil {
return fmt.Errorf("failed putting the lifecycle event hook execution status. the status was %s", *params.Status)
}
return nil
}
func handle() error {
functionName := os.Getenv("FUNCTION_NAME")
if functionName == "" {
return fmt.Errorf("FUNCTION_NAME not set")
}
log.Printf("function name: %s", functionName)
// invoke lambda via sdk
input := &lambdaService.InvokeInput{
FunctionName: &functionName,
Payload: nil,
LogType: aws.String(lambdaService.LogTypeTail), // returns the log in the response
InvocationType: aws.String(lambdaService.InvocationTypeRequestResponse), // synchronous - default
}
err := input.Validate()
if err != nil {
return fmt.Errorf("validating the input failed: %v", err)
}
resp, err := lambdaSvc.Invoke(input)
if err != nil {
return fmt.Errorf("failed to invoke lambda: %v", err)
}
decodeString, err := base64.StdEncoding.DecodeString(*resp.LogResult)
if err != nil {
return fmt.Errorf("failed to decode the log: %v", err)
}
log.Printf("log result: %s", decodeString)
if resp.FunctionError != nil {
return fmt.Errorf("lambda was invoked but returned error: %s", *resp.FunctionError)
}
return nil
}
func main() {
sess, err := session.NewSession()
if err != nil {
return
}
svc = codedeploy.New(sess)
lambdaSvc = lambdaService.New(sess)
lambda.Start(handler)
}
I want to send a Slack notification with an attached file. This is my current code:
package Message
import (
"fmt"
"os"
"github.com/ashwanthkumar/slack-go-webhook"
)
func Message(message string, cannalul string, attash bool) {
f, err := os.Open(filename)
if err != nil {
return false
}
defer f.Close()
_ = f
fullName := "myServer"
webhookUrl := "https://hooks.slack.com/services/......."
attachment1 := slack.Attachment {}
//attachment1.AddField(slack.Field { Title: "easySmtp", Value: "EasySmtp" }).AddField(slack.Field { Title: "Status", Value: "Completed" })
if attash {
attachment1.AddField(slack.Field { Title: "easySmtp", Value: fullName})
}
payload := slack.Payload {
Text: message,
Username: "worker",
Channel: cannalul,
IconEmoji: ":grin:",
Attachments: []slack.Attachment{attachment1},
}
err := slack.Send(webhookUrl, "", payload)
if len(err) > 0 {
fmt.Printf("error: %s\n", err)
}
}
My code works, but I don't know how I can add an attached file in my current code. How I can do this?
You can not attach a file to an attachment through a webhook in Slack. That functionality does not exist in Slack.
If its just text you can add the content as part of the message or another attachments (up to a limit of currently 500,000 characters, which will soon be reduced to 40,000 - see here for reference).
Or you can directly upload a file to a channel with the API method files.upload.