Ruby Gems C extension example not working - ruby

I am trying to follow this tutorial on building c extension in ruby gems http://guides.rubygems.org/gems-with-extensions/.
I have the following files:
ext/my_malloc/extconf.rb
require "mkmf"
abort "missing malloc()" unless have_func "malloc"
abort "missing free()" unless have_func "free"
create_makefile "my_malloc/my_malloc"
ext/my_malloc/my_malloc.c
#include <ruby.h>
struct my_malloc {
size_t size;
void *ptr;
};
static void
my_malloc_free(void *p) {
struct my_malloc *ptr = p;
if (ptr->size > 0)
free(ptr->ptr);
}
static VALUE
my_malloc_alloc(VALUE klass) {
VALUE obj;
struct my_malloc *ptr;
obj = Data_Make_Struct(klass, struct my_malloc, NULL, my_malloc_free, ptr);
ptr->size = 0;
ptr->ptr = NULL;
return obj;
}
static VALUE
my_malloc_init(VALUE self, VALUE size) {
struct my_malloc *ptr;
size_t requested = NUM2SIZET(size);
if (0 == requested)
rb_raise(rb_eArgError, "unable to allocate 0 bytes");
Data_Get_Struct(self, struct my_malloc, ptr);
ptr->ptr = malloc(requested);
if (NULL == ptr->ptr)
rb_raise(rb_eNoMemError, "unable to allocate %ld bytes", requested);
ptr->size = requested;
return self;
}
static VALUE
my_malloc_release(VALUE self) {
struct my_malloc *ptr;
Data_Get_Struct(self, struct my_malloc, ptr);
if (0 == ptr->size)
return self;
ptr->size = 0;
free(ptr->ptr);
return self;
}
void
Init_my_malloc(void) {
VALUE cMyMalloc;
cMyMalloc = rb_const_get(rb_cObject, rb_intern("MyMalloc"));
rb_define_alloc_func(cMyMalloc, my_malloc_alloc);
rb_define_method(cMyMalloc, "initialize", my_malloc_init, 1);
rb_define_method(cMyMalloc, "free", my_malloc_release, 0);
}
I do the following to build the extension:
$ cd ext/my_malloc
$ ruby extconf.rb
checking for malloc()... yes
checking for free()... yes
creating Makefile
$ make
compiling my_malloc.c
linking shared-object my_malloc.bundle
$ cd ../..
$ ruby -Ilib:ext -r my_malloc -e "p MyMalloc.new(5).free"
However, I get the following error on the last command:
/Users/Daniel/.rbenv/versions/2.1.6/lib/ruby/2.1.0/rubygems/core_ext/kernel_require.rb:55:in `require': cannot load such file -- my_malloc (LoadError)
from
/Users/Daniel/.rbenv/versions/2.1.6/lib/ruby/2.1.0/rubygems/core_ext/kernel_require.rb:55:in `require'
Note, I'm using rbenv, I've tried running rbenv rehash and I'm running version 2.1.6.

The correct shell command is:
ruby -Iext/my_malloc -r my_malloc -e "p MyMalloc.new(5).free"
The code also has some other mistakes. Correct
rb_raise(rb_eNoMemError, "unable to allocate %ld bytes", requested);
to
rb_raise(rb_eNoMemError, "unable to allocate %" PRIuSIZE " bytes", requested);
since requested has type size_t and not long (PRIuSIZE is non-standard and defined in "ruby/ruby.h"). And
cMyMalloc = rb_const_get(rb_cObject, rb_intern("MyMalloc"));
to
cMyMalloc = rb_define_class("MyMalloc", rb_cObject);
to actually define the MyMalloc class.

Related

Missing header <fluent-bit.h> while compiling C program

I'm trying to compile my C program using library Api of fluent-bit , but the header fluent-bit.h is missing and I don't understand why.
I installed fluent-bit using installation guide
Here is my code I want to test:
#include <fluent-bit.h>
int main()
{
int i;
int n;
char tmp[256];
flb_ctx_t *ctx;
int in_ffd;
int out_ffd;
/* Initialize library */
ctx = flb_create();
if (!ctx) {
exit(EXIT_FAILURE);
}
in_ffd = flb_input(ctx, "lib", NULL);
flb_input_set(ctx, in_ffd, "tag", "test", NULL);
out_ffd = flb_output(ctx, "stdout", NULL);
flb_output_set(ctx, out_ffd, "match", "test", NULL);
/* Start the background worker */
flb_start(ctx);
/* Push some data */
for (i = 0; i < 100; i++) {
n = snprintf(tmp, sizeof(tmp) - 1,
"[%f, {\"key\": \"val %i\"}]",
flb_time_now(), i);
flb_lib_push(ctx, in_ffd, tmp, n);
}
flb_stop(ctx);
/* Release Resources */
flb_destroy(ctx);
return 0;
}
Here the error I got:
hello.c:1:24: fatal error: fluent-bit.h: No such file or directory
#include <fluent-bit.h>
^
compilation terminated.
Problem solved, I didn't install fluent-bit and headers properly.
Here was the problem : the headers were missing, so move on to cd /path/to/downloaded/fluent-bit-x.y.z/includes
Then use
sudo cmake .
sudo make install
You'll get an output saying that the headers had been installed on your system.
To make sure fluent-bit is correctly installed too :
cd ..
sudo cmake .
sudo make install
You can now use the fluent-bit API without problems

How to chunk shell script input by time, not by size?

In a bash script I am using a many-producer single-consumer pattern. Producers are background processes writing lines into a fifo (via GNU Parallel). The consumer reads all lines from the fifo, then sorts, filters, and prints the formatted result to stdout.
However, it could take a long time until the full result is available. Producers are usually fast on the first few results but then would slow down. Here I am more interested to see chunks of data every few seconds, each sorted and filtered individually.
mkfifo fifo
parallel ... >"$fifo" &
while chunk=$(read with timeout 5s and at most 10s <"$fifo"); do
process "$chunk"
done
The loop would run until all producers are done and all input is read. Each chunk is read until there has been no new data for 5s, or until 10s have passed since the chunk was started. A chunk may also be empty if there was no new data for 10s.
I tried to make it work like this:
output=$(mktemp)
while true; do
wasTimeout=0 interruptAt=$(( $(date '+%s') + 10 ))
while true; do
IFS= read -r -t5 <>"${fifo}"
rc="$?"
if [[ "${rc}" -gt 0 ]]; then
[[ "${rc}" -gt 128 ]] && wasTimeout=1
break
fi
echo "$REPLY" >>"${output}"
if [[ $(date '+%s') -ge "${interruptAt}" ]]; then
wasTimeout=1
break
fi
done
echo '---' >>"${output}"
[[ "${wasTimeout}" -eq 0 ]] && break
done
Tried some variations of this. In the form above it reads the first chunk but then loops forever. If I use <"${fifo}" (no read/write as above) it blocks after the first chunk. Maybe all of this could be simplified with buffer and/or stdbuf? But both of them define blocks by size, not by time.
This is not a trivial problem to resolve. As I hinted, a C program (or a program in some programming language other than the shell) is probably the best solution. Some of the complicating factors are:
Reading with timeouts.
If data arrives soon enough, the timeout changes.
Different systems have different sets of interval timing functions:
alarm() is likely available everywhere, but has only 1-second resolution which is liable to accumulated rounding errors. (Compile this version with make UFLAGS=-DUSE_ALARM; on macOS, use make UFLAGS=-DUSE_ALARM LDLIB2=.)
setitimer()
uses microsecond timing and the struct timeval type. (Compile this version with make UFLAGS=-DUSE_SETITIMER; on macOS, compile with make UFLAGS=-DUSE_SETITIMER LDLIB2=.)
timer_create() and
timer_settime() etc use the modern nanosecond type struct timespec. This is available on Linux; it is not available on macOS 10.14.5 Mojave or earlier. (Compile this version with make; it won't work on macOS.)
The program usage message is:
$ chunker79 -h
Usage: chunker79 [-hvV][-c chunk][-d delay][-f file]
-c chunk Maximum time to wait for data in a chunk (default 10)
-d delay Maximum delay after line read (default: 5)
-f file Read from file instead of standard input
-h Print this help message and exit
-v Verbose mode: print timing information to stderr
-V Print version information and exit
$
This code is available in my SOQ (Stack Overflow Questions) repository on GitHub as file chunker79.c in the src/so-5631-4784 sub-directory. You will need some of the support code from the src/libsoq directory too.
/*
#(#)File: chunker79.c
#(#)Purpose: Chunk Reader for SO 5631-4784
#(#)Author: J Leffler
#(#)Copyright: (C) JLSS 2019
*/
/*TABSTOP=4*/
/*
** Problem specification from the Stack Overflow question
**
** In a bash script I am using a many-producer single-consumer pattern.
** Producers are background processes writing lines into a fifo (via GNU
** Parallel). The consumer reads all lines from the fifo, then sorts,
** filters, and prints the formatted result to stdout.
**
** However, it could take a long time until the full result is
** available. Producers are usually fast on the first few results but
** then would slow down. Here I am more interested to see chunks of
** data every few seconds, each sorted and filtered individually.
**
** mkfifo fifo
** parallel ... >"$fifo" &
** while chunk=$(read with timeout 5s and at most 10s <"$fifo"); do
** process "$chunk"
** done
**
** The loop would run until all producers are done and all input is
** read. Each chunk is read until there has been no new data for 5s, or
** until 10s have passed since the chunk was started. A chunk may also
** be empty if there was no new data for 10s.
*/
/*
** Analysis
**
** 1. If no data arrives at all for 10 seconds, then the program should
** terminate producing no output. This timeout is controlled by the
** value of time_chunk in the code.
** 2. If data arrives more or less consistently, then the collection
** should continue for 10s and then finish. This timeout is also
** controlled by the value of time_chunk in the code.
** 3. If a line of data arrives before 5 seconds have elapsed, and no
** more arrives for 5 seconds, then the collection should finish.
** (If the first line arrives after 5 seconds and no more arrives
** for more than 5 seconds, then the 10 second timeout cuts in.)
** This timeout is controlled by the value of time_delay in the code.
** 4. This means that we want two separate timers at work:
** - Chunk timer (started when the program starts).
** - Delay timer (started each time a line is read).
**
** It doesn't matter which timer goes off, but further timer signals
** should be ignored. External signals will confuse things; tough!
**
** -- Using alarm(2) is tricky because it provides only one time, not two.
** -- Using getitimer(2), setitimer(2) uses obsolescent POSIX functions,
** but these are available on macOS.
** -- Using timer_create(2), timer_destroy(2), timer_settime(2),
** timer_gettime(2) uses current POSIX function but is not available
** on macOS.
*/
#include "posixver.h"
#include "stderr.h"
#include "timespec_io.h"
#include <assert.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/uio.h>
#include <time.h>
#include <unistd.h>
#ifdef USE_SETITIMER
#include "timeval_math.h"
#include "timeval_io.h"
#include <sys/time.h>
#endif /* USE_SETITIMER */
static const char optstr[] = "hvVc:d:f:";
static const char usestr[] = "[-hvV][-c chunk][-d delay][-f file]";
static const char hlpstr[] =
" -c chunk Maximum time to wait for data in a chunk (default 10)\n"
" -d delay Maximum delay after line read (default: 5)\n"
" -f file Read from file instead of standard input\n"
" -h Print this help message and exit\n"
" -v Verbose mode: print timing information to stderr\n"
" -V Print version information and exit\n"
;
static struct timespec time_delay = { .tv_sec = 5, .tv_nsec = 0 };
static struct timespec time_chunk = { .tv_sec = 10, .tv_nsec = 0 };
static struct timespec time_start;
static bool verbose = false;
static void set_chunk_timeout(void);
static void set_delay_timeout(void);
static void cancel_timeout(void);
static void alarm_handler(int signum);
// Using signal() manages to set SA_RESTART on a Mac.
// This is allowed by standard C and POSIX, sadly.
// signal(SIGALRM, alarm_handler);
#if defined(USE_ALARM)
static void set_chunk_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
alarm(time_chunk.tv_sec);
struct sigaction sa;
sa.sa_handler = alarm_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sigaction(SIGALRM, &sa, NULL);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void set_delay_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
unsigned time_left = alarm(0);
if (time_left > time_delay.tv_sec)
alarm(time_delay.tv_sec);
else
alarm(time_left);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void cancel_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
alarm(0);
signal(SIGALRM, SIG_IGN);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
#elif defined(USE_SETITIMER)
static inline struct timeval cvt_timespec_to_timeval(struct timespec ts)
{
return (struct timeval){ .tv_sec = ts.tv_sec, .tv_usec = ts.tv_nsec / 1000 };
}
static void set_chunk_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerval tv_new = { { 0, 0 }, { 0, 0 } };
tv_new.it_value = cvt_timespec_to_timeval(time_chunk);
struct itimerval tv_old;
if (setitimer(ITIMER_REAL, &tv_new, &tv_old) != 0)
err_syserr("failed to set interval timer: ");
struct sigaction sa;
sa.sa_handler = alarm_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sigaction(SIGALRM, &sa, NULL);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void set_delay_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerval tv_until;
if (getitimer(ITIMER_REAL, &tv_until) != 0)
err_syserr("failed to set interval timer: ");
struct timeval tv_delay = cvt_timespec_to_timeval(time_delay);
if (verbose)
{
char buff1[32];
fmt_timeval(&tv_delay, 6, buff1, sizeof(buff1));
char buff2[32];
fmt_timeval(&tv_until.it_value, 6, buff2, sizeof(buff2));
err_remark("---- %s(): delay %s, left %s\n", __func__, buff1, buff2);
}
if (cmp_timeval(tv_until.it_value, tv_delay) <= 0)
{
if (verbose)
err_remark("---- %s(): no need for delay timer\n", __func__);
}
else
{
struct itimerval tv_new = { { 0, 0 }, { 0, 0 } };
tv_new.it_value = cvt_timespec_to_timeval(time_delay);
struct itimerval tv_old;
if (setitimer(ITIMER_REAL, &tv_new, &tv_old) != 0)
err_syserr("failed to set interval timer: ");
if (verbose)
err_remark("---- %s(): set delay timer\n", __func__);
}
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void cancel_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerval tv_new =
{
.it_value = { .tv_sec = 0, .tv_usec = 0 },
.it_interval = { .tv_sec = 0, .tv_usec = 0 },
};
struct itimerval tv_old;
if (setitimer(ITIMER_REAL, &tv_new, &tv_old) != 0)
err_syserr("failed to set interval timer: ");
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
#else /* USE_TIMER_GETTIME */
#include "timespec_math.h"
static timer_t t0 = { 0 };
static void set_chunk_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct sigevent ev =
{
.sigev_notify = SIGEV_SIGNAL,
.sigev_signo = SIGALRM,
.sigev_value.sival_int = 0,
.sigev_notify_function = 0,
.sigev_notify_attributes = 0,
};
if (timer_create(CLOCK_REALTIME, &ev, &t0) < 0)
err_syserr("failed to create a timer: ");
struct itimerspec it =
{
.it_interval = { .tv_sec = 0, .tv_nsec = 0 },
.it_value = time_chunk,
};
struct itimerspec ot;
if (timer_settime(t0, 0, &it, &ot) != 0)
err_syserr("failed to activate timer: ");
struct sigaction sa;
sa.sa_handler = alarm_handler;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sigaction(SIGALRM, &sa, NULL);
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void set_delay_timeout(void)
{
if (verbose)
err_remark("-->> %s()\n", __func__);
struct itimerspec time_until;
if (timer_gettime(t0, &time_until) != 0)
err_syserr("failed to set per-process timer: ");
char buff1[32];
fmt_timespec(&time_delay, 6, buff1, sizeof(buff1));
char buff2[32];
fmt_timespec(&time_until.it_value, 6, buff2, sizeof(buff2));
err_remark("---- %s(): delay %s, left %s\n", __func__, buff1, buff2);
if (cmp_timespec(time_until.it_value, time_delay) <= 0)
{
if (verbose)
err_remark("---- %s(): no need for delay timer\n", __func__);
}
else
{
struct itimerspec time_new =
{
.it_interval = { .tv_sec = 0, .tv_nsec = 0 },
.it_value = time_delay,
};
struct itimerspec time_old;
if (timer_settime(t0, 0, &time_new, &time_old) != 0)
err_syserr("failed to set per-process timer: ");
if (verbose)
err_remark("---- %s(): set delay timer\n", __func__);
}
if (verbose)
err_remark("<<-- %s()\n", __func__);
}
static void cancel_timeout(void)
{
if (timer_delete(t0) != 0)
err_syserr("failed to delete timer: ");
}
#endif /* Timing mode */
/* Writing to stderr via err_remark() is not officially supported */
static void alarm_handler(int signum)
{
assert(signum == SIGALRM);
if (verbose)
err_remark("---- %s(): signal %d\n", __func__, signum);
}
static void read_chunks(FILE *fp)
{
size_t num_data = 0;
size_t max_data = 0;
struct iovec *data = 0;
size_t buflen = 0;
char *buffer = 0;
ssize_t length;
size_t chunk_len = 0;
clock_gettime(CLOCK_REALTIME, &time_start);
set_chunk_timeout();
while ((length = getline(&buffer, &buflen, fp)) != -1)
{
if (num_data >= max_data)
{
size_t new_size = (num_data * 2) + 2;
void *newspace = realloc(data, new_size * sizeof(data[0]));
if (newspace == 0)
err_syserr("failed to allocate %zu bytes data: ", new_size * sizeof(data[0]));
data = newspace;
max_data = new_size;
}
data[num_data].iov_base = buffer;
data[num_data].iov_len = length;
num_data++;
if (verbose)
err_remark("Received line %zu\n", num_data);
chunk_len += length;
buffer = 0;
buflen = 0;
set_delay_timeout();
}
cancel_timeout();
if (chunk_len > 0)
{
if ((length = writev(STDOUT_FILENO, data, num_data)) < 0)
err_syserr("failed to write %zu bytes to standard output: ", chunk_len);
else if ((size_t)length != chunk_len)
err_error("failed to write %zu bytes to standard output "
"(short write of %zu bytes)\n", chunk_len, (size_t)length);
}
if (verbose)
err_remark("---- %s(): data written (%zu bytes)\n", __func__, length);
for (size_t i = 0; i < num_data; i++)
free(data[i].iov_base);
free(data);
free(buffer);
}
int main(int argc, char **argv)
{
const char *name = "(standard input)";
FILE *fp = stdin;
err_setarg0(argv[0]);
err_setlogopts(ERR_MICRO);
int opt;
while ((opt = getopt(argc, argv, optstr)) != -1)
{
switch (opt)
{
case 'c':
if (scn_timespec(optarg, &time_chunk) != 0)
err_error("Failed to convert '%s' into a time value\n", optarg);
break;
case 'd':
if (scn_timespec(optarg, &time_delay) != 0)
err_error("Failed to convert '%s' into a time value\n", optarg);
break;
case 'f':
if ((fp = fopen(optarg, "r")) == 0)
err_syserr("Failed to open file '%s' for reading: ", optarg);
name = optarg;
break;
case 'h':
err_help(usestr, hlpstr);
/*NOTREACHED*/
case 'v':
verbose = true;
break;
case 'V':
err_version("CHUNKER79", &"#(#)$Revision$ ($Date$)"[4]);
/*NOTREACHED*/
default:
err_usage(usestr);
/*NOTREACHED*/
}
}
if (optind != argc)
err_usage(usestr);
if (verbose)
{
err_remark("chunk: %3lld.%09ld\n", (long long)time_chunk.tv_sec, time_chunk.tv_nsec);
err_remark("delay: %3lld.%09ld\n", (long long)time_delay.tv_sec, time_delay.tv_nsec);
err_remark("file: %s\n", name);
}
read_chunks(fp);
return 0;
}
My SOQ repository also has a script gen-data.sh which makes use of some custom programs to generate a data stream such as this (the seed value is written to standard error, not standard output):
$ gen-data.sh
# Seed: 1313715286
2019-06-03 23:04:16.653: Zunmieoprri Rdviqymcho 5878 2017-03-29 03:59:15 Udransnadioiaeamprirteo
2019-06-03 23:04:18.525: Rndflseoevhgs Etlaevieripeoetrnwkn 9500 2015-12-18 10:49:15 Ebyrcoebeezatiagpleieoefyc
2019-06-03 23:04:20.526: Nrzsuiakrooab Nbvliinfqidbujoops 1974 2020-05-13 08:05:14 Lgithearril
2019-06-03 23:04:21.777: Eeagop Aieneose 6533 2016-11-06 22:51:58 Aoejlwebbssroncmeovtuuueigraa
2019-06-03 23:04:23.876: Izirdoeektau Atesltiybysaclee 4557 2020-09-13 02:24:46 Igrooiaauiwtna
2019-06-03 23:04:26.145: Yhioit Eamrexuabagsaraiw 9703 2014-09-13 07:44:12 Dyiiienglolqopnrbneerltnmsdn
^C
$
When fed into chunker79 with default options, I get output like:
$ gen-data.sh | chunker79
# Seed: 722907235
2019-06-03 23:06:20.570: Aluaezkgiebeewal Oyvahee 1022 2015-08-12 07:45:54 Weuababeeduklleym
2019-06-03 23:06:24.100: Gmujvoyevihvoilc Negeiiuvleem 8196 2015-08-29 21:15:15 Nztkrvsadeoeagjgoyotvertavedi
$
If you analyze the time intervals (look at the first two fields in the output lines), that output meets the specification. A still more detailed analysis is shown by:
$ timecmd -mr -- gen-data.sh | timecmd -mr -- chunker79
2019-06-03 23:09:14.246 [PID 57159] gen-data.sh
2019-06-03 23:09:14.246 [PID 57160] chunker79
# Seed: -1077610201
2019-06-03 23:09:14.269: Woreio Rdtpimvoscttbyhxim 7893 2017-03-12 12:46:57 Uywaietirkekes
2019-06-03 23:09:16.939: Uigaba Nzoxdeuisofai 3630 2017-11-16 09:28:59 Jnsncgoesycsevdscugoathusaoq
2019-06-03 23:09:17.845: Sscreua Aloaoonnsuur 5163 2016-08-13 19:47:15 Injhsiifqovbnyeooiimitaaoir
2019-06-03 23:09:19.272 [PID 57160; status 0x0000] - 5.026s - chunker79
2019-06-03 23:09:22.084 [PID 57159; status 0x8D00] - 7.838s - gen-data.sh
$
There is a noticeable pause in this setup between when the output from chunker79 appears and when gen-data.sh completes. That's due to Bash waiting on all processes in the pipeline to complete, and gen-data.sh doesn't complete until the next time it writes to the pipe after the message that finishes chunker79. This is an artefact of this test setup; it wouldn't be a factor in the shell script outlined in the question.
I would consider writing a safe multi-threaded program with queues.
I know Java better, but there might be more modern suitable languages like Go and Kotlin.
Something like this:
#!/usr/bin/perl
$timeout = 3;
while(<STDIN>) {
# Make sure there is some input
push #out,$_;
eval {
local $SIG{ALRM} = sub { die };
alarm $timeout;
while(<STDIN>) {
alarm $timeout;
push #out,$_;
}
alarm 0;
};
system "echo","process",#out;
}
GNU Parallel 20200122 introduced --blocktimeout (--bt):
find ~ | parallel -j3 --bt 2s --pipe wc
This works like normal GNU Parallel except if it takes > 2 seconds to fill a block. In that case the block read so far is simply passed to wc (unless it is empty).
It has a slightly odd startup behaviour: You have to wait 3*2s (jobslots*timeout) before the output stabilizes, and you get an output at least every 2s.

JeMalloc does not create memory leak dump

I need help with memory profiling using JeMalloc.
I do the following things:
git clone https://github.com/jemalloc/jemalloc
cd jemalloc
./autogen.sh --enable-perf
make dist
make
sudo make install
export MALLOC_CONF=prof_leak:true,lg_prof_sample:0,prof_final:true
export LD_PRELOAD=/usr/local/Cellar/jemalloc/5.1.0/lib/libjemalloc.dylib
Then I run my application:
./some_executed_file
It is 100% that this binary file will use jemalloc
Because when I call
typedef struct {
char *cur;
char *end;
} MallocStatus;
static void GetJemallocStatus(void *mstat_arg, const char *status) {
MallocStatus *mstat = reinterpret_cast<MallocStatus *>(mstat_arg);
size_t status_len = status ? strlen(status) : 0;
size_t buf_size = (size_t)(mstat->end - mstat->cur);
if (!status_len || status_len > buf_size) {
return;
}
snprintf(mstat->cur, buf_size, "%s", status);
mstat->cur += status_len;
}
MallocStatus mstat;
const unsigned int kMallocStatusLen = 1000000;
std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
mstat.cur = buf.get();
mstat.end = buf.get() + kMallocStatusLen;
je_malloc_stats_print(GetJemallocStatus, &mstat, "");
stats->append(buf.get());
I see JeMalloc statistics.
Regarding to
https://github.com/jemalloc/jemalloc/wiki/Use-Case:-Leak-Checking
I do everything correct - but I still don't see jeprof dump file do analyze memory leaks.
Thanks in advance.
Try adding prof:true,prof_active:true to your MALLOC_CONF, and using --enable-prof flag instead of --enable-perf.

wcstombs & allocating memory for character array on heap

I'm reading a file with a single wide character line in it. But, I never know how long it is going to be. I've read this into a std::wstring, inString, and have managed to create the multi byte string out of thin air (Q1 - are these called r-values?). Q2 - Now, how do I allocate memory for this in the heap and obtain a smart pointer to it ? I do not want to use new or malloc (and call free or delete eventually) or any constant to store it on the stack (for I can never know the max length). Q3 - Can I make use of the make_shared or make_unique function templates here ? Q4 - To be specific, can I get a pointer like shared_ptr<char> pointing to the char array allocated on the heap ?
I tried something like the following,
std::shared_ptr<char> MBString(const_cast<char*>(std::string(inString.begin(), inString.end()).c_str()));
it did not work. I tried a few suggestions on the internet but I don't know how to do it yet.
Q5 - Let alone Wide char to multi -byte conversion, in general, how do I allocate an arbitrary length char string on the heap and get a smart pointer to it ?
std::wfstream inFile(L"lengthUnkown.txt", std::ios::in);
std::wstring inString;
inFile >> inString;
std::wcout << inString << std::endl; //prints correctly
std::cout << (const_cast<char*>(std::string(inString.begin(), inString.end()).c_str())) << std::endl; //this prints the line correctly as expected
//convert wide character string to multi-byte on the heap pointed, to by MBString
//std::cout << MBString << std::endl; //I want to print the multi-byte string like this
return 0;
Not resource optimal but reliable:
wchar_t* mb2wstr(const char* inval) {
size_t size = std::strlen(inval);
#define OUTSZ (size+1)*sizeof(wchar_t)
auto buf = (wchar_t*)std::malloc(OUTSZ);
std::memset(buf, 0, OUTSZ);
std::setlocale(LC_CTYPE,""); // необходима, чтобы отработала "mbstowcs"
size = std::mbstowcs(buf, inval, size);
if ( size == (size_t)(-1) ) {
std::free(buf);
buf = nullptr;
} else {
buf = (wchar_t*)std::realloc(buf,OUTSZ);
}
return buf;
#undef OUTSZ
}
char* wstr2mb(const wchar_t* inval) {
size_t size = std::wcslen(inval);
#define OUTSZ (size+1)*MB_CUR_MAX // Maximum length of a multibyte character in the current locale
auto buf = (char*)std::malloc(OUTSZ);
std::memset(buf, 0, OUTSZ);
std::setlocale(LC_CTYPE,""); // необходима, чтобы отработала "wcstombs"
size = std::wcstombs(buf, inval, size*sizeof(wchar_t));
if ( size == (size_t)(-1) ) {
std::free(buf);
buf = nullptr;
} else {
buf = (char*)std::realloc(buf,size+1);
}
return buf;
#undef OUTSZ
}
const std::string pwchar2string(const wchar_t* inval) {
char* tmp = wstr2mb(inval);
string out{tmp};
std::free(tmp);
return out;
}
const std::wstring pchar2wstring(const char* inval) {
wchar_t* tmp = mb2wstr(inval);
wstring out{tmp};
std::free(tmp);
return out;
}
const wstring string2wstring(const string& value) {
return pchar2wstring(value.c_str());
}
const string wstring2string(const wstring& value) {
return pwchar2string(value.c_str());
}
const wchar_t* char2wchar(const char* value) {
return pchar2wstring(value).c_str();
}
const char* wchar2char(const wchar_t* value) {
return pwchar2string(value).c_str();
}

Why pam_loginuid module fails on writing to /proc/self/loginuid with -EPERM?

I found that application using pam library to authenticate fails on error:
Error writing /proc/self/loginuid: Operation not permitted
By strace i found that fail is on write to the /proc/self/loginuid file.
Further inspection and adding some debug code to kernel (code below):
static ssize_t proc_loginuid_write(struct file * file, const char __user * buf,
size_t count, loff_t *ppos)
{
struct inode * inode = file_inode(file);
uid_t loginuid;
kuid_t kloginuid;
int rv;
printk(KERN_DEBUG "proc_loginuid_write\n");
printk(KERN_DEBUG "a+++ %s\n", current->comm);
printk(KERN_DEBUG "b+++ %s\n", pid_task(proc_pid(inode), PIDTYPE_PID)->comm);
printk(KERN_DEBUG "+++2++ pid = %d\n", current->pid);
printk(KERN_DEBUG "+++3++ pid = %d\n", pid_task(proc_pid(inode), PIDTYPE_PID)->pid);
rcu_read_lock();
if (current != pid_task(proc_pid(inode), PIDTYPE_PID)) {
rcu_read_unlock();
printk(KERN_ERR "proc_loginuid_write failed by permission!\n");
return -EPERM;
}
rcu_read_unlock();
if (*ppos != 0) {
/* No partial writes. */
return -EINVAL;
}
rv = kstrtou32_from_user(buf, count, 10, &loginuid);
if (rv < 0)
return rv;
/* is userspace tring to explicitly UNSET the loginuid? */
if (loginuid == AUDIT_UID_UNSET) {
kloginuid = INVALID_UID;
} else {
kloginuid = make_kuid(file->f_cred->user_ns, loginuid);
if (!uid_valid(kloginuid))
return -EINVAL;
}
rv = audit_set_loginuid(kloginuid);
if (rv < 0)
return rv;
return count;
}
showed in dmesg that:
[ 30.672242] proc_loginuid_write
[ 30.672249] a+++ testapp
[ 30.672251] b+++ testapp
[ 30.672254] +++2++ pid = 2920
[ 30.672257] +++3++ pid = 2451
[ 30.672259] proc_loginuid_write failed by permission!
Name testapp is intentionally changed name. So it looks like the file /proc/self/loginuid is file created by parent, and it is read by child thread.
I tested same code on kernel 3.14 and 4.9 and on 3.14 kernel it works and on kernel 4.9 it doesn't works. Why?
I found the solution for the problem.
Old kernel 3.14 has turned off option CONFIG_AUDITSYSCALL in config. So on there was no file /proc/self/loginuid and pam module simply don't cares when there is no such file.
On newer kernel 4.9 option is automatically selected by CONFIG_AUDIT=y.
So simplest solution is to turn off CONFIG_AUDIT option, but why in process of kernel evolution CONFIG_AUDITSYSCALL became a non controllable option is matter for other question.
Thanks!

Resources