GNUPG how to encrypt file without it's original extension in the name? - gnupg

I need to automatically encrypt file e.g. myfile.xls to myfile.gpg.
Now when I'm trying to encrypt it automatically or with GNU privacy assistent, the encrypted file is myfile.xls.gpg. When I delete that .xls, it has no extension after decrypting.
Thank you.

That gpgencrypt.c option looks like far too much effort.
To encrypt do this:
gpg -er $recipient_id -o myfile.gpg myfile.xls
To decrypt and restore the original filename do this:
gpg --use-embedded-filename -d myfile.gpg
That name has been encrypted and is stored with the encrypted data. That flag tells GPG to use the original filename instead of creating a new filename based on the name of the encrypted file.

This could be done easily with a simple C app that controls the output. The below file can do what you are looking for, you just need to compile it (linked against libgpgme) and then pass it the input file, the desired output file and the key-ids to use for encryption.
To compile using gcc, simply save the code below as gpgencrypt.c, and compile with gcc gpgencrypt.c -lgpgme -o gpgencrypt. (you need libgpgme installed)
The invocation syntax is gpgencrypt <input file> <output file> <encrypt to>. i.e. gpgencrypt myfile.xls myfile.gpg 01234567 12345678
file: gpgencrypt.c
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <stdlib.h>
#include <errno.h>
#include <locale.h>
#include <gpgme.h>
#define fail_if_err(err) \
do { \
if (err) { \
fprintf (stderr, "%s:%d: %s: %s\n", \
__FILE__, __LINE__, gpgme_strsource (err), \
gpgme_strerror (err)); \
exit (1); \
} \
} \
while (0)
typedef char * string;
void gpgEncrypt(const char *fileToEncrypt, const char *outputFileName,
char *encryptTo[],
int encryptToLength) {
gpgme_ctx_t ctx;
gpgme_error_t err;
gpgme_data_t in, out;
gpgme_encrypt_result_t enc_result;
FILE *outputFile;
gpgme_key_t keys[encryptToLength];
int nrecipients = 0;
int BUF_SIZE = 512;
char buf[BUF_SIZE + 1];
int ret;
/* Begin setup of GPGME */
gpgme_check_version (NULL);
setlocale (LC_ALL, "");
gpgme_set_locale (NULL, LC_CTYPE, setlocale (LC_CTYPE, NULL));
#ifndef HAVE_W32_SYSTEM
gpgme_set_locale (NULL, LC_MESSAGES, setlocale (LC_MESSAGES, NULL));
#endif
/* End setup of GPGME */
// Create the GPGME Context
err = gpgme_new (&ctx);
// Error handling
fail_if_err (err);
// Set the context to textmode
gpgme_set_textmode (ctx, 1);
// Enable ASCII armor on the context
gpgme_set_armor (ctx, 1);
// Create a data object pointing to the input file
err = gpgme_data_new_from_file (&in, fileToEncrypt, 1);
// Error handling
fail_if_err (err);
// Create a data object pointing to the out buffer
err = gpgme_data_new (&out);
// Error handling
fail_if_err (err);
// Retrieve the keys used to encrypt to
for (nrecipients=0; nrecipients < encryptToLength; nrecipients++) {
printf("Retrieving key: %s, %i of %i\n", encryptTo[nrecipients], nrecipients, encryptToLength);
err = gpgme_get_key (ctx, encryptTo[nrecipients], &keys[nrecipients], 0);
if(err != GPG_ERR_NO_ERROR)
printf("error");
}
// NULL terminate the key array
keys[nrecipients] = NULL;
// Encrypt the contents of "in" using the defined mode and place it into "out"
err = gpgme_op_encrypt (ctx, keys, GPGME_ENCRYPT_ALWAYS_TRUST, in, out);
// Error handling
fail_if_err (err);
// Retrieve the encrypt result from the context
enc_result = gpgme_op_encrypt_result (ctx);
// Check for invalid recipients
if (enc_result->invalid_recipients) {
fail_if_err (err);
}
// Open the output file
outputFile = fopen (outputFileName, "w+");
// Rewind the "out" data object
ret = gpgme_data_seek (out, 0, SEEK_SET);
// Error handling
if (ret)
fail_if_err (gpgme_err_code_from_errno (errno));
// Read the contents of "out" and place it into buf
while ((ret = gpgme_data_read (out, buf, BUF_SIZE)) > 0) {
// Write the contents of "buf" to "outputFile"
fwrite (buf, ret, 1, outputFile);
}
// Error handling
if (ret < 0)
fail_if_err (gpgme_err_code_from_errno (errno));
// Close "outputFile"
fclose(outputFile);
// Unreference the key objects
for (nrecipients=0; nrecipients < sizeof(&keys) / sizeof(int) + 1; nrecipients++) {
if (keys[nrecipients]) {
printf ("Releasing key: %s\n", keys[nrecipients]->subkeys->fpr);
gpgme_key_unref (keys[nrecipients]);
}
}
// Release the "in" data object
gpgme_data_release (in);
// Release the "out" data object
gpgme_data_release (out);
// Release the context
gpgme_release (ctx);
}
int
main (int argc, char **argv) {
if (argc < 4) {
printf ("Usage: gpgencrypt <input file> <output file> <encrypt to>\n");
exit (1);
}
int pos, n;
string encryptToList[argc - 4];
printf ("Encrypting %s and placing the result into %s\n",
argv[1], argv[2]);
for (pos=3; pos < argc; pos++) {
encryptToList[n] = argv[pos];
n++;
}
gpgEncrypt (argv[1], argv[2], encryptToList, argc - 3);
return 0;
}

Related

How to chunk shell script input by time, not by size?

In a bash script I am using a many-producer single-consumer pattern. Producers are background processes writing lines into a fifo (via GNU Parallel). The consumer reads all lines from the fifo, then sorts, filters, and prints the formatted result to stdout.
However, it could take a long time until the full result is available. Producers are usually fast on the first few results but then would slow down. Here I am more interested to see chunks of data every few seconds, each sorted and filtered individually.
mkfifo fifo
parallel ... >"$fifo" &
while chunk=$(read with timeout 5s and at most 10s <"$fifo"); do
process "$chunk"
done
The loop would run until all producers are done and all input is read. Each chunk is read until there has been no new data for 5s, or until 10s have passed since the chunk was started. A chunk may also be empty if there was no new data for 10s.
I tried to make it work like this:
output=$(mktemp)
while true; do
wasTimeout=0 interruptAt=$(( $(date '+%s') + 10 ))
while true; do
IFS= read -r -t5 <>"${fifo}"
rc="$?"
if [[ "${rc}" -gt 0 ]]; then
[[ "${rc}" -gt 128 ]] && wasTimeout=1
break
fi
echo "$REPLY" >>"${output}"
if [[ $(date '+%s') -ge "${interruptAt}" ]]; then
wasTimeout=1
break
fi
done
echo '---' >>"${output}"
[[ "${wasTimeout}" -eq 0 ]] && break
done
Tried some variations of this. In the form above it reads the first chunk but then loops forever. If I use <"${fifo}" (no read/write as above) it blocks after the first chunk. Maybe all of this could be simplified with buffer and/or stdbuf? But both of them define blocks by size, not by time.
This is not a trivial problem to resolve. As I hinted, a C program (or a program in some programming language other than the shell) is probably the best solution. Some of the complicating factors are:
Reading with timeouts.
If data arrives soon enough, the timeout changes.
Different systems have different sets of interval timing functions:
alarm() is likely available everywhere, but has only 1-second resolution which is liable to accumulated rounding errors. (Compile this version with make UFLAGS=-DUSE_ALARM; on macOS, use make UFLAGS=-DUSE_ALARM LDLIB2=.)
setitimer()
uses microsecond timing and the struct timeval type. (Compile this version with make UFLAGS=-DUSE_SETITIMER; on macOS, compile with make UFLAGS=-DUSE_SETITIMER LDLIB2=.)
timer_create() and
timer_settime() etc use the modern nanosecond type struct timespec. This is available on Linux; it is not available on macOS 10.14.5 Mojave or earlier. (Compile this version with make; it won't work on macOS.)
The program usage message is:
$ chunker79 -h
Usage: chunker79 [-hvV][-c chunk][-d delay][-f file]
-c chunk Maximum time to wait for data in a chunk (default 10)
-d delay Maximum delay after line read (default: 5)
-f file Read from file instead of standard input
-h Print this help message and exit
-v Verbose mode: print timing information to stderr
-V Print version information and exit
$
This code is available in my SOQ (Stack Overflow Questions) repository on GitHub as file chunker79.c in the src/so-5631-4784 sub-directory. You will need some of the support code from the src/libsoq directory too.
/*
#(#)File: chunker79.c
#(#)Purpose: Chunk Reader for SO 5631-4784
#(#)Author: J Leffler
#(#)Copyright: (C) JLSS 2019
*/
/*TABSTOP=4*/
/*
** Problem specification from the Stack Overflow question
**
** In a bash script I am using a many-producer single-consumer pattern.
** Producers are background processes writing lines into a fifo (via GNU
** Parallel). The consumer reads all lines from the fifo, then sorts,
** filters, and prints the formatted result to stdout.
**
** However, it could take a long time until the full result is
** available. Producers are usually fast on the first few results but
** then would slow down. Here I am more interested to see chunks of
** data every few seconds, each sorted and filtered individually.
**
** mkfifo fifo
** parallel ... >"$fifo" &
** while chunk=$(read with timeout 5s and at most 10s <"$fifo"); do
** process "$chunk"
** done
**
** The loop would run until all producers are done and all input is
** read. Each chunk is read until there has been no new data for 5s, or
** until 10s have passed since the chunk was started. A chunk may also
** be empty if there was no new data for 10s.
*/
/*
** Analysis
**
** 1. If no data arrives at all for 10 seconds, then the program should
** terminate producing no output. This timeout is controlled by the
** value of time_chunk in the code.
** 2. If data arrives more or less consistently, then the collection
** should continue for 10s and then finish. This timeout is also
** controlled by the value of time_chunk in the code.
** 3. If a line of data arrives before 5 seconds have elapsed, and no
** more arrives for 5 seconds, then the collection should finish.
** (If the first line arrives after 5 seconds and no more arrives
** for more than 5 seconds, then the 10 second timeout cuts in.)
** This timeout is controlled by the value of time_delay in the code.
** 4. This means that we want two separate timers at work:
** - Chunk timer (started when the program starts).
** - Delay timer (started each time a line is read).
**
** It doesn't matter which timer goes off, but further timer signals
** should be ignored. External signals will confuse things; tough!
**
** -- Using alarm(2) is tricky because it provides only one time, not two.
** -- Using getitimer(2), setitimer(2) uses obsolescent POSIX functions,
** but these are available on macOS.
** -- Using timer_create(2), timer_destroy(2), timer_settime(2),
** timer_gettime(2) uses current POSIX function but is not available
** on macOS.
*/
#include "posixver.h"
#include "stderr.h"
#include "timespec_io.h"
#include <assert.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/uio.h>
#include <time.h>
#include <unistd.h>
#ifdef USE_SETITIMER
#include "timeval_math.h"
#include "timeval_io.h"
#include <sys/time.h>
#endif /* USE_SETITIMER */
static const char optstr[] = "hvVc:d:f:";
static const char usestr[] = "[-hvV][-c chunk][-d delay][-f file]";
static const char hlpstr[] =
" -c chunk Maximum time to wait for data in a chunk (default 10)\n"
" -d delay Maximum delay after line read (default: 5)\n"
" -f file Read from file instead of standard input\n"
" -h Print this help message and exit\n"
" -v Verbose mode: print timing information to stderr\n"
" -V Print version information and exit\n"
;
static struct timespec time_delay = { .tv_sec = 5, .tv_nsec = 0 };
static struct timespec time_chunk = { .tv_sec = 10, .tv_nsec = 0 };
static struct timespec time_start;
static bool verbose = false;
static void set_chunk_timeout(void);
static void set_delay_timeout(void);
static void cancel_timeout(void);
static void alarm_handler(int signum);
// Using signal() manages to set SA_RESTART on a Mac.
// This is allowed by standard C and POSIX, sadly.
// signal(SIGALRM, alarm_handler);
#if defined(USE_ALARM)
static void set_chunk_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
alarm(time_chunk.tv_sec);
struct sigaction sa;
sa.sa_handler = alarm_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sigaction(SIGALRM, &sa, NULL);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void set_delay_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
unsigned time_left = alarm(0);
if (time_left > time_delay.tv_sec)
alarm(time_delay.tv_sec);
else
alarm(time_left);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void cancel_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
alarm(0);
signal(SIGALRM, SIG_IGN);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
#elif defined(USE_SETITIMER)
static inline struct timeval cvt_timespec_to_timeval(struct timespec ts)
{
return (struct timeval){ .tv_sec = ts.tv_sec, .tv_usec = ts.tv_nsec / 1000 };
}
static void set_chunk_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerval tv_new = { { 0, 0 }, { 0, 0 } };
tv_new.it_value = cvt_timespec_to_timeval(time_chunk);
struct itimerval tv_old;
if (setitimer(ITIMER_REAL, &tv_new, &tv_old) != 0)
err_syserr("failed to set interval timer: ");
struct sigaction sa;
sa.sa_handler = alarm_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sigaction(SIGALRM, &sa, NULL);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void set_delay_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerval tv_until;
if (getitimer(ITIMER_REAL, &tv_until) != 0)
err_syserr("failed to set interval timer: ");
struct timeval tv_delay = cvt_timespec_to_timeval(time_delay);
if (verbose)
{
char buff1[32];
fmt_timeval(&tv_delay, 6, buff1, sizeof(buff1));
char buff2[32];
fmt_timeval(&tv_until.it_value, 6, buff2, sizeof(buff2));
err_remark("---- %s(): delay %s, left %s\n", __func__, buff1, buff2);
}
if (cmp_timeval(tv_until.it_value, tv_delay) <= 0)
{
if (verbose)
err_remark("---- %s(): no need for delay timer\n", __func__);
}
else
{
struct itimerval tv_new = { { 0, 0 }, { 0, 0 } };
tv_new.it_value = cvt_timespec_to_timeval(time_delay);
struct itimerval tv_old;
if (setitimer(ITIMER_REAL, &tv_new, &tv_old) != 0)
err_syserr("failed to set interval timer: ");
if (verbose)
err_remark("---- %s(): set delay timer\n", __func__);
}
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void cancel_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerval tv_new =
{
.it_value = { .tv_sec = 0, .tv_usec = 0 },
.it_interval = { .tv_sec = 0, .tv_usec = 0 },
};
struct itimerval tv_old;
if (setitimer(ITIMER_REAL, &tv_new, &tv_old) != 0)
err_syserr("failed to set interval timer: ");
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
#else /* USE_TIMER_GETTIME */
#include "timespec_math.h"
static timer_t t0 = { 0 };
static void set_chunk_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct sigevent ev =
{
.sigev_notify = SIGEV_SIGNAL,
.sigev_signo = SIGALRM,
.sigev_value.sival_int = 0,
.sigev_notify_function = 0,
.sigev_notify_attributes = 0,
};
if (timer_create(CLOCK_REALTIME, &ev, &t0) < 0)
err_syserr("failed to create a timer: ");
struct itimerspec it =
{
.it_interval = { .tv_sec = 0, .tv_nsec = 0 },
.it_value = time_chunk,
};
struct itimerspec ot;
if (timer_settime(t0, 0, &it, &ot) != 0)
err_syserr("failed to activate timer: ");
struct sigaction sa;
sa.sa_handler = alarm_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sigaction(SIGALRM, &sa, NULL);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void set_delay_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerspec time_until;
if (timer_gettime(t0, &time_until) != 0)
err_syserr("failed to set per-process timer: ");
char buff1[32];
fmt_timespec(&time_delay, 6, buff1, sizeof(buff1));
char buff2[32];
fmt_timespec(&time_until.it_value, 6, buff2, sizeof(buff2));
err_remark("---- %s(): delay %s, left %s\n", __func__, buff1, buff2);
if (cmp_timespec(time_until.it_value, time_delay) <= 0)
{
if (verbose)
err_remark("---- %s(): no need for delay timer\n", __func__);
}
else
{
struct itimerspec time_new =
{
.it_interval = { .tv_sec = 0, .tv_nsec = 0 },
.it_value = time_delay,
};
struct itimerspec time_old;
if (timer_settime(t0, 0, &time_new, &time_old) != 0)
err_syserr("failed to set per-process timer: ");
if (verbose)
err_remark("---- %s(): set delay timer\n", __func__);
}
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void cancel_timeout(void)
{
if (timer_delete(t0) != 0)
err_syserr("failed to delete timer: ");
}
#endif /* Timing mode */
/* Writing to stderr via err_remark() is not officially supported */
static void alarm_handler(int signum)
{
assert(signum == SIGALRM);
if (verbose)
err_remark("---- %s(): signal %d\n", __func__, signum);
}
static void read_chunks(FILE *fp)
{
size_t num_data = 0;
size_t max_data = 0;
struct iovec *data = 0;
size_t buflen = 0;
char *buffer = 0;
ssize_t length;
size_t chunk_len = 0;
clock_gettime(CLOCK_REALTIME, &time_start);
set_chunk_timeout();
while ((length = getline(&buffer, &buflen, fp)) != -1)
{
if (num_data >= max_data)
{
size_t new_size = (num_data * 2) + 2;
void *newspace = realloc(data, new_size * sizeof(data[0]));
if (newspace == 0)
err_syserr("failed to allocate %zu bytes data: ", new_size * sizeof(data[0]));
data = newspace;
max_data = new_size;
}
data[num_data].iov_base = buffer;
data[num_data].iov_len = length;
num_data++;
if (verbose)
err_remark("Received line %zu\n", num_data);
chunk_len += length;
buffer = 0;
buflen = 0;
set_delay_timeout();
}
cancel_timeout();
if (chunk_len > 0)
{
if ((length = writev(STDOUT_FILENO, data, num_data)) < 0)
err_syserr("failed to write %zu bytes to standard output: ", chunk_len);
else if ((size_t)length != chunk_len)
err_error("failed to write %zu bytes to standard output "
"(short write of %zu bytes)\n", chunk_len, (size_t)length);
}
if (verbose)
err_remark("---- %s(): data written (%zu bytes)\n", __func__, length);
for (size_t i = 0; i < num_data; i++)
free(data[i].iov_base);
free(data);
free(buffer);
}
int main(int argc, char **argv)
{
const char *name = "(standard input)";
FILE *fp = stdin;
err_setarg0(argv[0]);
err_setlogopts(ERR_MICRO);
int opt;
while ((opt = getopt(argc, argv, optstr)) != -1)
{
switch (opt)
{
case 'c':
if (scn_timespec(optarg, &time_chunk) != 0)
err_error("Failed to convert '%s' into a time value\n", optarg);
break;
case 'd':
if (scn_timespec(optarg, &time_delay) != 0)
err_error("Failed to convert '%s' into a time value\n", optarg);
break;
case 'f':
if ((fp = fopen(optarg, "r")) == 0)
err_syserr("Failed to open file '%s' for reading: ", optarg);
name = optarg;
break;
case 'h':
err_help(usestr, hlpstr);
/*NOTREACHED*/
case 'v':
verbose = true;
break;
case 'V':
err_version("CHUNKER79", &"#(#)$Revision$ ($Date$)"[4]);
/*NOTREACHED*/
default:
err_usage(usestr);
/*NOTREACHED*/
}
}
if (optind != argc)
err_usage(usestr);
if (verbose)
{
err_remark("chunk: %3lld.%09ld\n", (long long)time_chunk.tv_sec, time_chunk.tv_nsec);
err_remark("delay: %3lld.%09ld\n", (long long)time_delay.tv_sec, time_delay.tv_nsec);
err_remark("file: %s\n", name);
}
read_chunks(fp);
return 0;
}
My SOQ repository also has a script gen-data.sh which makes use of some custom programs to generate a data stream such as this (the seed value is written to standard error, not standard output):
$ gen-data.sh
# Seed: 1313715286
2019-06-03 23:04:16.653: Zunmieoprri Rdviqymcho 5878 2017-03-29 03:59:15 Udransnadioiaeamprirteo
2019-06-03 23:04:18.525: Rndflseoevhgs Etlaevieripeoetrnwkn 9500 2015-12-18 10:49:15 Ebyrcoebeezatiagpleieoefyc
2019-06-03 23:04:20.526: Nrzsuiakrooab Nbvliinfqidbujoops 1974 2020-05-13 08:05:14 Lgithearril
2019-06-03 23:04:21.777: Eeagop Aieneose 6533 2016-11-06 22:51:58 Aoejlwebbssroncmeovtuuueigraa
2019-06-03 23:04:23.876: Izirdoeektau Atesltiybysaclee 4557 2020-09-13 02:24:46 Igrooiaauiwtna
2019-06-03 23:04:26.145: Yhioit Eamrexuabagsaraiw 9703 2014-09-13 07:44:12 Dyiiienglolqopnrbneerltnmsdn
^C
$
When fed into chunker79 with default options, I get output like:
$ gen-data.sh | chunker79
# Seed: 722907235
2019-06-03 23:06:20.570: Aluaezkgiebeewal Oyvahee 1022 2015-08-12 07:45:54 Weuababeeduklleym
2019-06-03 23:06:24.100: Gmujvoyevihvoilc Negeiiuvleem 8196 2015-08-29 21:15:15 Nztkrvsadeoeagjgoyotvertavedi
$
If you analyze the time intervals (look at the first two fields in the output lines), that output meets the specification. A still more detailed analysis is shown by:
$ timecmd -mr -- gen-data.sh | timecmd -mr -- chunker79
2019-06-03 23:09:14.246 [PID 57159] gen-data.sh
2019-06-03 23:09:14.246 [PID 57160] chunker79
# Seed: -1077610201
2019-06-03 23:09:14.269: Woreio Rdtpimvoscttbyhxim 7893 2017-03-12 12:46:57 Uywaietirkekes
2019-06-03 23:09:16.939: Uigaba Nzoxdeuisofai 3630 2017-11-16 09:28:59 Jnsncgoesycsevdscugoathusaoq
2019-06-03 23:09:17.845: Sscreua Aloaoonnsuur 5163 2016-08-13 19:47:15 Injhsiifqovbnyeooiimitaaoir
2019-06-03 23:09:19.272 [PID 57160; status 0x0000] - 5.026s - chunker79
2019-06-03 23:09:22.084 [PID 57159; status 0x8D00] - 7.838s - gen-data.sh
$
There is a noticeable pause in this setup between when the output from chunker79 appears and when gen-data.sh completes. That's due to Bash waiting on all processes in the pipeline to complete, and gen-data.sh doesn't complete until the next time it writes to the pipe after the message that finishes chunker79. This is an artefact of this test setup; it wouldn't be a factor in the shell script outlined in the question.
I would consider writing a safe multi-threaded program with queues.
I know Java better, but there might be more modern suitable languages like Go and Kotlin.
Something like this:
#!/usr/bin/perl
$timeout = 3;
while(<STDIN>) {
# Make sure there is some input
push #out,$_;
eval {
local $SIG{ALRM} = sub { die };
alarm $timeout;
while(<STDIN>) {
alarm $timeout;
push #out,$_;
}
alarm 0;
};
system "echo","process",#out;
}
GNU Parallel 20200122 introduced --blocktimeout (--bt):
find ~ | parallel -j3 --bt 2s --pipe wc
This works like normal GNU Parallel except if it takes > 2 seconds to fill a block. In that case the block read so far is simply passed to wc (unless it is empty).
It has a slightly odd startup behaviour: You have to wait 3*2s (jobslots*timeout) before the output stabilizes, and you get an output at least every 2s.

ESP32 SDCARD speed issue

I am trying to use SD Card on Wrover kit but it seems that speed is the big issue on the board itself.
At first I wanted to download file from the net and save it to SDCARD but it took too long so to test it I've written a loop to save some chars into file and create ~1MB large file on a SDCARD and it takes forever.
What could be the cause that ~1MB file could be such a long task to do.
I have combined two examples into one to do the task.
Also I have commented everything and left just part with writing to file to demonstrate issue.
/* ESP HTTP Client Example
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#define LOG_LOCAL_LEVEL ESP_LOG_ERROR
#include <string.h>
#include <stdlib.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "esp_system.h"
#include "nvs_flash.h"
#include "app_wifi.h"
#include "esp_http_client.h"
/* SD CARD */
#include "esp_vfs_fat.h"
#include "driver/sdmmc_host.h"
#include "driver/sdspi_host.h"
#include "sdmmc_cmd.h"
/* SD CARD */
#define USE_SPI_MODE
#ifdef USE_SPI_MODE
// Pin mapping when using SPI mode.
// With this mapping, SD card can be used both in SPI and 1-line SD mode.
// Note that a pull-up on CS line is required in SD mode.
#define PIN_NUM_MISO 2
#define PIN_NUM_MOSI 15
#define PIN_NUM_CLK 14
#define PIN_NUM_CS 13
#endif //USE_SPI_MODE
//#define MAX_HTTP_RECV_BUFFER 512
#define MAX_HTTP_RECV_BUFFER 1024
static const char *TAG = "HTTP_CLIENT";
// ------------------ GLOBAL VARS -----------------------------
FILE *fp=NULL;
// ------------------ GLOBAL VARS -----------------------------
/* Root cert for howsmyssl.com, taken from howsmyssl_com_root_cert.pem
The PEM file was extracted from the output of this command:
openssl s_client -showcerts -connect www.howsmyssl.com:443 </dev/null
The CA root cert is the last cert given in the chain of certs.
To embed it in the app binary, the PEM file is named
in the component.mk COMPONENT_EMBED_TXTFILES variable.
*/
extern const char howsmyssl_com_root_cert_pem_start[] asm("_binary_howsmyssl_com_root_cert_pem_start");
extern const char howsmyssl_com_root_cert_pem_end[] asm("_binary_howsmyssl_com_root_cert_pem_end");
esp_err_t _http_event_handler(esp_http_client_event_t *evt)
{
switch(evt->event_id) {
case HTTP_EVENT_ERROR:
ESP_LOGE (TAG, "HTTP_EVENT_ERROR");
break;
case HTTP_EVENT_ON_CONNECTED:
ESP_LOGE (TAG, "HTTP_EVENT_ON_CONNECTED");
break;
case HTTP_EVENT_HEADER_SENT:
ESP_LOGE (TAG, "HTTP_EVENT_HEADER_SENT");
break;
case HTTP_EVENT_ON_HEADER:
ESP_LOGE (TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
break;
case HTTP_EVENT_ON_DATA:
ESP_LOGE (TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
if (!esp_http_client_is_chunked_response(evt->client)) {
// Write out data
// printf("%.*s", evt->data_len, (char*)evt->data);
if(fp == NULL){
fp = fopen("/sdcard/muzika.mp3","wb");
}
if(fp != NULL){
fwrite(evt->data,1,evt->data_len,fp);
ESP_LOGE (TAG, "---------- HTTP_EVENT_ON_DATA - WRITING TO SD CARD ----------");
}
}
break;
case HTTP_EVENT_ON_FINISH:
ESP_LOGE (TAG, "HTTP_EVENT_ON_FINISH");
fclose(fp);
fp = NULL;
break;
case HTTP_EVENT_DISCONNECTED:
ESP_LOGE (TAG, "HTTP_EVENT_DISCONNECTED");
break;
}
return ESP_OK;
}
static void http_download_chunk()
{
esp_http_client_config_t config = {
//.url = "http://httpbin.org/stream-bytes/8912",
.url = "http://www.theoctopusproject.com/mp3/whatthey.mp3",
.event_handler = _http_event_handler,
};
esp_http_client_handle_t client = esp_http_client_init(&config);
esp_err_t err = esp_http_client_perform(client);
if (err == ESP_OK) {
ESP_LOGI(TAG, "HTTP chunk encoding Status = %d, content_length = %d",
esp_http_client_get_status_code(client),
esp_http_client_get_content_length(client));
} else {
ESP_LOGE(TAG, "Error perform http request %s", esp_err_to_name(err));
}
esp_http_client_cleanup(client);
}
static void http_test_task(void *pvParameters)
{
app_wifi_wait_connected();
ESP_LOGI(TAG, "Connected to AP, begin http example");
http_download_chunk();
ESP_LOGI(TAG, "Finish http example");
vTaskDelete(NULL);
}
void app_main()
{
esp_err_t ret = nvs_flash_init();
if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
ESP_ERROR_CHECK(nvs_flash_erase());
ret = nvs_flash_init();
}
ESP_ERROR_CHECK(ret);
// -- app_wifi_initialise();
/* SDCARD SETUP */
#ifndef USE_SPI_MODE
ESP_LOGI(TAG, "Using SDMMC peripheral");
sdmmc_host_t host = SDMMC_HOST_DEFAULT();
// This initializes the slot without card detect (CD) and write protect (WP) signals.
// Modify slot_config.gpio_cd and slot_config.gpio_wp if your board has these signals.
sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();
// To use 1-line SD mode, uncomment the following line:
// slot_config.width = 1;
// GPIOs 15, 2, 4, 12, 13 should have external 10k pull-ups.
// Internal pull-ups are not sufficient. However, enabling internal pull-ups
// does make a difference some boards, so we do that here.
gpio_set_pull_mode(15, GPIO_PULLUP_ONLY); // CMD, needed in 4- and 1- line modes
gpio_set_pull_mode(2, GPIO_PULLUP_ONLY); // D0, needed in 4- and 1-line modes
gpio_set_pull_mode(4, GPIO_PULLUP_ONLY); // D1, needed in 4-line mode only
gpio_set_pull_mode(12, GPIO_PULLUP_ONLY); // D2, needed in 4-line mode only
gpio_set_pull_mode(13, GPIO_PULLUP_ONLY); // D3, needed in 4- and 1-line modes
#else
ESP_LOGI(TAG, "Using SPI peripheral");
sdmmc_host_t host = SDSPI_HOST_DEFAULT();
sdspi_slot_config_t slot_config = SDSPI_SLOT_CONFIG_DEFAULT();
slot_config.gpio_miso = PIN_NUM_MISO;
slot_config.gpio_mosi = PIN_NUM_MOSI;
slot_config.gpio_sck = PIN_NUM_CLK;
slot_config.gpio_cs = PIN_NUM_CS;
// This initializes the slot without card detect (CD) and write protect (WP) signals.
// Modify slot_config.gpio_cd and slot_config.gpio_wp if your board has these signals.
#endif //USE_SPI_MODE
// Options for mounting the filesystem.
// If format_if_mount_failed is set to true, SD card will be partitioned and
// formatted in case when mounting fails.
esp_vfs_fat_sdmmc_mount_config_t mount_config = {
.format_if_mount_failed = false,
.max_files = 5,
.allocation_unit_size = 16 * 1024
};
// Use settings defined above to initialize SD card and mount FAT filesystem.
// Note: esp_vfs_fat_sdmmc_mount is an all-in-one convenience function.
// Please check its source code and implement error recovery when developing
// production applications.
sdmmc_card_t* card;
ret = esp_vfs_fat_sdmmc_mount("/sdcard", &host, &slot_config, &mount_config, &card);
if (ret != ESP_OK) {
if (ret == ESP_FAIL) {
ESP_LOGE(TAG, "Failed to mount filesystem. "
"If you want the card to be formatted, set format_if_mount_failed = true.");
} else {
ESP_LOGE(TAG, "Failed to initialize the card (%s). "
"Make sure SD card lines have pull-up resistors in place.", esp_err_to_name(ret));
}
return;
}
// Card has been initialized, print its properties
sdmmc_card_print_info(stdout, card);
// Use POSIX and C standard library functions to work with files.
// First create a file.
ESP_LOGE(TAG, "Opening file");
FILE* f = fopen("/sdcard/hello.txt", "w");
if (f == NULL) {
ESP_LOGE(TAG, "Failed to open file for writing");
return;
}
// ------------------------ WRITE INTO FILE -------------------
int j=0;
for(int i = 0 ; i < 1048576 ; i++,j++){
//fprintf(f, "Hello %s!\n", card->cid.name);
fprintf(f, "X");
if(j > 1024){
vTaskDelay(3);
j = 0;
}
}
fclose(f);
ESP_LOGE(TAG, "File written");
// ------------------------ WRITE INTO FILE -------------------
//-- xTaskCreate(&http_test_task, "http_test_task", 8192, NULL, 5, NULL);
//xTaskCreate(&http_test_task, "http_test_task", 16384, NULL, 5, NULL);
}
Your example is writing one byte at a time. That won't get you an accurate idea of how the overall speed of writing to the SD card - your code will spend a disproportionate amount of time processing the individual writes.
If you want to get a more accurate idea of how fast you can write to the card, try writing 1024 bytes (or even more) at a time. Benchmarks that measure maximum throughput and performance-sensitive applications will always write as much data as they can at a time in order to minimize the overhead of the write.
Also, don't use fprintf() if possible. It has extra overhead because it parses the format string. Try fwrite() - it will have less overhead and will give you a better idea of what you can expect in the best case.
Try something like this instead:
char *buf = malloc(1024);
if(buf) {
for(int i = 0; i < 1024; i++) {
fwrite(buf, 1, 1024, f);
vTaskDelay(3); // you'll speed this up if you can omit this
}
ESP_LOGE(TAG, "File written");
free(buf);
} else
ESP_LOGE(TAG, "malloc() failed");
fclose(f);

FFT of samples from portAudio stream

Beginner here, (OSX 10.9.5, Xcode 6)
I have a portAudio stream that gives out noise. Now I'd like to get those random values generated in the callback and run them through an fftw plan. As far as I know, fftw needs to be executed in the main. So how can I show the numbers from the callback to the main? I have a feeling it has something to do with pointers but that's a very uneducated guess...
I'm having some difficulty with joining two different libraries. Little help would be greatly appreciated, thank you!
#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include "portaudio.h"
#include "fftw3.h"
#define NUM_SECONDS (1)
#define SAMPLE_RATE (44100)
typedef struct
{
float left_phase;
float right_phase;
}
paTestData;
static int patestCallback( const void *inputBuffer, void *outputBuffer,
unsigned long framesPerBuffer,
const PaStreamCallbackTimeInfo* timeInfo,
PaStreamCallbackFlags statusFlags,
void *userData )
{
/* Cast data passed through stream to our structure. */
paTestData *data = (paTestData*)userData;
float *out = (float*)outputBuffer;
unsigned int i;
(void) inputBuffer; /* Prevent unused variable warning. */
for( i=0; i<framesPerBuffer; i++ )
{
*out++ = data->left_phase; /* left */
*out++ = data->right_phase; /* right */
/* Generate random value that ranges between -1.0 and 1.0. */
data->left_phase = (((float)rand()/(float)(RAND_MAX)) * 2) - 1 ;
data->right_phase = (((float)rand()/(float)(RAND_MAX)) * 2) - 1 ;
printf("%f, %f\n", data->left_phase, data->right_phase);
}
return 0;
}
/*******************************************************************/
static paTestData data;
int main(void);
int main(void)
{
PaStream *stream;
PaError err;
printf("PortAudio Test: output noise.\n");
/* Initialize our data for use by callback. */
data.left_phase = data.right_phase = 0.0;
/* Initialize library before making any other calls. */
err = Pa_Initialize();
if( err != paNoError ) goto error;
/* Open an audio I/O stream. */
err = Pa_OpenDefaultStream( &stream,
0, /* no input channels */
2, /* stereo output */
paFloat32, /* 32 bit floating point output */
SAMPLE_RATE,
512, /* frames per buffer */
patestCallback,
&data );
if( err != paNoError ) goto error;
err = Pa_StartStream( stream );
if( err != paNoError ) goto error;
/* Sleep for several seconds. */
Pa_Sleep(NUM_SECONDS*1000);
err = Pa_StopStream( stream );
if( err != paNoError ) goto error;
err = Pa_CloseStream( stream );
if( err != paNoError ) goto error;
Pa_Terminate();
printf("Test finished.\n");
return err;
error:
Pa_Terminate();
fprintf( stderr, "An error occured while using the portaudio stream\n" );
fprintf( stderr, "Error number: %d\n", err );
fprintf( stderr, "Error message: %s\n", Pa_GetErrorText( err ) );
return err;
}
You could try running the stream in "blocking write" mode instead of using the callback. To use this mode, you pass NULL for the streamCallback parameter of Pa_OpenDefaultStream and then you continually call Pa_WriteStream in a loop. The call will block as necessary. Something like this pseudo code:
Pa_OpenStream(&stream, 0, 2, paFloat32, SAMPLE_RATE, 512, NULL, NULL);
Pa_StartStream(stream);
float interleavedSamples[2*512];
for (int i = 0 ; i < SAMPLE_RATE/512 ; i++) // approx 1 second
{
GenerateNoise(&interleavedSamples, 2, 512, &data);
RunFft(interleavedSamples, ...);
PaWriteStream(stream, interleavedSamples, 512);
}

zlib deflate getting Z_DATA_ERROR

I'm trying to deflate and inflate a text file using zlib-1.2.7 with VC++2010.
I used pipe.c as basis and rewrote the main() in order to set the input/output files.
I'm able to deflate any text file but i can't inflate large files without getting a Z_DATA_ERROR (-3).
the inf() and def() are not changed from pipe.c.
Here is my main() :
int main (){
FILE *a, *b, *c;
int ret;
//ZIP
a = fopen("a_data.txt", "r");
b = fopen("b_compressedData.zip", "w");
if(a != NULL && b != NULL){
ret = def(a, b, Z_DEFAULT_COMPRESSION);
printf("%d\n", ret);
if (ret != Z_OK) zerr(ret);
fclose(a);
fclose(b);
}
//UNZIP
b = fopen("b_compressedData.zip", "r");
c = fopen("c_uncompressedData.txt", "w");
if(c != NULL && b != NULL){
ret = inf(b, c);
printf("%d\n", ret);
if (ret != Z_OK) zerr(ret);
fclose(b);
fclose(c);
}
return 0;
}
Here is the inf() function :
int inf(FILE *source, FILE *dest)
{
int ret;
unsigned have;
z_stream strm;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
/* allocate inflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
ret = inflateInit(&strm);
if (ret != Z_OK)
return ret;
/* decompress until deflate stream ends or end of file */
do {
strm.avail_in = fread(in, 1, CHUNK, source);
if (ferror(source)) {
(void)inflateEnd(&strm);
return Z_ERRNO;
}
if (strm.avail_in == 0)
break;
strm.next_in = in;
/* run inflate() on input until output buffer not full */
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR; /* and fall through */
case Z_DATA_ERROR:
case Z_MEM_ERROR:
(void)inflateEnd(&strm);
return ret;
}
have = CHUNK - strm.avail_out;
if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
(void)inflateEnd(&strm);
return Z_ERRNO;
}
} while (strm.avail_out == 0);
/* done when inflate() says it's done */
} while (ret != Z_STREAM_END);
/* clean up and return */
(void)inflateEnd(&strm);
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}
You need to add a "b" in the modes for fopen(), where the b is for binary. E.g. "rb" and "wb". Otherwise on Windows you get end-of-line conversions which mess up binary data.
By the way, you should not call the compressed file .zip since it isn't. zlib writes zlib-formatted data, or if requested gzip-formatted data. zlib does not write zip-formatted data on its own. There is third-party code for writing zip files, such as in the contrib directory of the zlib source distribution, or in libzip.
I use .zz as the suffix for zlib-formatted data. .gz is the suffix for gzip-formatted data.

libusb data transfer

Hii I have to make a code to read data from usb drive which can be pendrive and later a data acquisition card . i have written this much of code which detects all usb connection n print their info. Altough i don't know how to proceed further . I m also confused as to reading data from say pendrive means as in opening some files or what? Please also tell how to find endpoint of device currently i'm jsut using hit n trial to find it .
PLEASE help me out .I have read whole documentation on synchronous and asynchronous I/O.
enter code here
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <libusb-1.0/libusb.h>
//=========================================================================
// This program detects usb and print out their details
//=========================================================================
int main (int argc, char *argv)
{
libusb_device **devList = NULL;
libusb_device *devPtr = NULL;
libusb_device_handle *devHandle = NULL;
libusb_context *ctx = NULL; //a libusb session
struct libusb_device_descriptor devDesc;
unsigned char strDesc[256];
ssize_t numUsbDevs = 0; // pre-initialized scalars
ssize_t idx = 0;
int retVal = 0;
//========================================================================
// test out the libusb functions
//========================================================================
printf ("*************************\n USB Detection Program:\n*************************\n");
retVal = libusb_init (&ctx);
if(retVal < 0) {
printf ("%d",retVal," Init Error "); //there was an error
return 1;
}
//========================================================================
// Get the list of USB devices visible to the system.
//========================================================================
numUsbDevs = libusb_get_device_list (ctx, &devList);
//========================================================================
// Loop through the list, looking for the device
//========================================================================
while (idx < numUsbDevs)
{
printf ("\n\n[%d]\n", idx+1);
//=====================================================================
// Get next device pointer out of the list, use it to open the device.
//=====================================================================
devPtr = devList[idx];
retVal = libusb_open (devPtr, &devHandle);
if (retVal != LIBUSB_SUCCESS)
break;
//=====================================================================
// Get the device descriptor for this device.
//=====================================================================
retVal = libusb_get_device_descriptor (devPtr, &devDesc);
if (retVal != LIBUSB_SUCCESS)
break;
//=====================================================================
// Get the string associated with iManufacturer index.
//=====================================================================
printf ("iManufacturer = %d", devDesc.iManufacturer);
if (devDesc.iManufacturer > 0)
{
retVal = libusb_get_string_descriptor_ascii
(devHandle, devDesc.iManufacturer, strDesc, 256);
if (retVal < 0)
break;
printf (" string = %s", strDesc);
}
//========================================================================
// Get string associated with iProduct index.
//========================================================================
printf (" \niProduct = %d", devDesc.iProduct);
if (devDesc.iProduct > 0)
{
retVal = libusb_get_string_descriptor_ascii
(devHandle, devDesc.iProduct, strDesc, 256);
if (retVal < 0)
break;
printf (" string = %s", strDesc);
}
//==================================================================
// Get string associated with iSerialNumber index.
//==================================================================
printf (" \niSerialNumber = %d", devDesc.iSerialNumber);
if (devDesc.iSerialNumber > 0)
{
retVal = libusb_get_string_descriptor_ascii
(devHandle, devDesc.iSerialNumber, strDesc, 256);
if (retVal < 0)
break;
printf (" string = %s", strDesc);
}
//==================================================================
// Print product id and Vendor id
//==================================================================
printf (" \nProductid = %d", devDesc.idProduct);
printf (" \nVendorid = %d", devDesc.idVendor);
//========================================================================
// Close and try next one.
//========================================================================
libusb_close (devHandle);
devHandle = NULL;
idx++;
//========================================================================
// Selection of device by user
//========================================================================
if(idx==numUsbDevs)
{ printf("\n\nselect the device : ");
scanf("%d",&idx);
if(idx > numUsbDevs)
{printf("Invalid input, Quitting.............");
break; }
devPtr = devList[idx-1];
retVal = libusb_open (devPtr, &devHandle);
if (retVal != LIBUSB_SUCCESS)
break;
retVal = libusb_get_device_descriptor (devPtr, &devDesc);
if (retVal != LIBUSB_SUCCESS)
break;
printf (" \nProductid = %d", devDesc.idProduct);
printf (" \nVendorid = %d", devDesc.idVendor);
unsigned char data[4] ; //data to read
//data[0]='a';data[1]='b';data[2]='c';data[3]='d'; //some dummy values
int r; //for return values
r = libusb_claim_interface(devHandle, 1); //claim interface 0 (the first) of device
if(r < 0) {
printf("\nCannot Claim Interface");
return 1;
}
printf("\nClaimed Interface");
int actual_length; //used to find out how many bytes were written
r = libusb_bulk_transfer(devHandle,LIBUSB_ENDPOINT_IN, data, 2, &actual_length, 0);
if (r == 0 && actual_length == sizeof(data)) {
// results of the transaction can now be found in the data buffer
// parse them here and report button press
}
else {
error();
}
r = libusb_release_interface(devHandle, 1); //release the claimed interface
if(r!=0) {
printf("\nCannot Release Interface");
return 1;
}
printf("\nReleased Interface");
idx=numUsbDevs +2;
}
} // end of while loop
if (devHandle != NULL)
{
//========================================================================
// Close device if left open due to break out of loop on error.
//========================================================================
libusb_close (devHandle);
}
libusb_exit (ctx); //close the session
printf ("\n*************************\n Done\n*************************\n");
return 0;
}
//==========================================
// EOF
//====================

Resources