I have leraned the boost sample about "Create vectors in shared_memory".
Now My data structure is like :
Data structure:
enum FuncIndex
{
enmFunc_glBegin,
...
}
class CGLParam {};
class Funcall
{
vector<CGLParam> vecParams;
};
class Global_Funcall
{
typedef allocator<CGLParam*, managed_shared_memory::segment_manager> ShmemAllocator;
typedef vector<CGLParam*, ShmemAllocator> MyVector;
MyVector<FunCall> vecFuncalls;
};
Global_Funcall()
{
shared_memory_object::remove("MySharedMemory");
managed_shared_memory segment(create_only, "MySharedMemory", 65536);
//Initialize shared memory STL-compatible allocator
const ShmemAllocator alloc_inst(segment.get_segment_manager());
//Construct a vector named "MyVector" in shared memory with argument alloc_inst
vecFuncalls= segment.construct<MyVector>("MyVector")(alloc_inst);
}
void InvokeFuncs(CGLParam *presult)
{
managed_shared_memory open_segment(open_only,"MySharedMemory");
listParams = open_segment.find<MyVector>("MyVector").first;
// MyVector::const_iterator it;
// for (it = listParams->cbegin(); it != listParams->cend(); it++)
// {
// (*it)->InvokeFunc(presult);
// }
}
My problem is "How to construct the vecParams and how to get it". the size of data is very big (opengl function calls)
The structure is use to save the opengl function calls.
Besides 'obvious' typos, you try to assign an IPC vector (MyVector*) to a standard vector in the GlobalFuncall constructor. That will never work. C++ is a strongly typed language, so the types have to match if you want to assign[1].
Besides this there seems to be a conceptual problem:
if the goal is to have a data collection that could be larger than fits in phyical memory, shared-memory per se isn't going to help. You'd want to look at memory-mapped files
if you want shared-memory because you can share it between processes (hence Boost Interprocess), you will need to think of process synchronization, or you will see complicated bugs because of data races.
you cannot safely store raw pointers inside this containers. Instead, store the actual elements there (or maybe look at bip::offset_ptr<> if you want to get really fancy).
Here's a 'fixed up' demonstration
fixing the C++ compilation issues,
changing the element type to be CGLParam instead of CGLParam*
fixing the member type to match the SHM vector and
adding basic shared mutex synchronization (this is an art in itself and you will want to read more about this)
See it Live On Coliru[1]
#include <vector>
#include <boost/interprocess/managed_mapped_file.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/sync/named_mutex.hpp>
#include <boost/interprocess/sync/named_recursive_mutex.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
namespace bip = boost::interprocess;
using mutex_type = bip::named_mutex;
class CGLParam {};
typedef bip::allocator<CGLParam, bip::managed_shared_memory::segment_manager> ShmemAllocator;
typedef std::vector<CGLParam, ShmemAllocator> MyVector;
class Funcall
{
std::vector<CGLParam> vecParams;
};
struct mutex_remove
{
mutex_remove() { mutex_type::remove("2faa9c3f-4cc0-49c5-8f79-f99ce5a5d526"); }
~mutex_remove(){ mutex_type::remove("2faa9c3f-4cc0-49c5-8f79-f99ce5a5d526"); }
} remover;
static mutex_type mutex(bip::open_or_create,"2faa9c3f-4cc0-49c5-8f79-f99ce5a5d526");
class Global_Funcall
{
MyVector* vecFuncalls;
Global_Funcall()
{
bip::scoped_lock<mutex_type> lock(mutex);
bip::shared_memory_object::remove("MySharedMemory");
bip::managed_shared_memory segment(bip::create_only, "MySharedMemory", 65536);
//Initialize shared memory STL-compatible allocator
const ShmemAllocator alloc_inst(segment.get_segment_manager());
//Construct a vector named "MyVector" in shared memory with argument alloc_inst
vecFuncalls = segment.construct<MyVector>("MyVector")(alloc_inst);
}
};
void InvokeFuncs(CGLParam *presult)
{
bip::scoped_lock<mutex_type> lock(mutex);
bip::managed_shared_memory open_segment(bip::open_only, "MySharedMemory");
auto listParams = open_segment.find<MyVector>("MyVector").first;
MyVector::const_iterator it;
for (it = listParams->cbegin(); it != listParams->cend(); it++)
{
//it->InvokeFunc(presult);
}
}
int main()
{
}
[1] Unless, of course, there's a suitable conversion
[2] Coliru doesn't support the required IPC mechanisms :/
Related
I'm trying to include <arpa/inet.h> in a low-level library so that I have access to hton* and ntoh* functions in the library. The low-level library gets called into by higher-level code running a Boost asio socket. I'm aware Boost asio contains the hton* and ntoh* functions, but i'd like to avoid linking all of Boost asio to the library since hton*/ntoh* are all I need.
However, if I simply include <arpa/inet.h> in the low-level library, 0 bytes always will be sent from the Boost asio socket. Confirmed by Wireshark.
Here's the class where i'd like to include <arpa/inet.h> but not Boost. If <arpa/inet.h> is included, 0 bytes will be sent.
#pragma pack(push, 1)
#include "PduHeader.h"
#include <arpa/inet.h>
class ClientInfoPdu
{
public:
ClientInfoPdu(const uint16_t _client_receiver_port)
{
set_client_receiver_port(_client_receiver_port);
}
PduHeader pdu_header{CLIENT_INFO_PDU, sizeof(client_receiver_port)};
inline void set_client_receiver_port(const uint16_t _client_receiver_port)
{
//client_receiver_port = htons(_client_receiver_port);
client_receiver_port = _client_receiver_port;
}
inline uint16_t get_client_receiver_port()
{
return client_receiver_port;
}
inline size_t get_total_size()
{
return sizeof(PduHeader) + pdu_header.get_pdu_payload_size();
}
private:
uint16_t client_receiver_port;
};
#pragma pack(pop)
Here's the higher level code that includes Boost and attempts to send the data via a socket. The printout indicates 5 bytes were sent, however 0 bytes were actually sent.
#include "ServerConnectionThread.h"
#include "config/ClientConfig.h"
#include "protocol_common/ClientInfoPdu.h"
#include <boost/asio.hpp>
#include <unistd.h>
using boost::asio::ip::udp;
void ServerConnectionThread::execute()
{
boost::asio::io_service io_service;
udp::endpoint remote_endpoint =
udp::endpoint(boost::asio::ip::address::from_string(SERVER_IP), SERVER_PORT);
udp::socket socket(io_service);
socket.open(udp::v4());
ClientInfoPdu client_info_pdu = ClientInfoPdu(RECEIVE_PORT);
while (true)
{
uint16_t total_size = client_info_pdu.get_total_size();
socket.send_to(boost::asio::buffer(&client_info_pdu, total_size), remote_endpoint);
printf("sent %u bytes\n", total_size);
usleep(1000000);
}
}
Again, simply removing "#include <arpa/inet.h>" will cause this code to function as expected and send 5 bytes per packet.
How is ClientInfoPdu defined? This looks like it is likely UB:
boost::asio::buffer(&client_info_pdu, total_size)
The thing is total size is sizeof(PduHeader) + pdu_header.get_pdu_payload_size() (so sizeof(PduHeader) + 2);
First problem is that you're mixing access modifiers, killing the POD/standard_layout properties of your types.
#include <type_traits>
static_assert(std::is_standard_layout_v<PduHeader> && std::is_trivial_v<PduHeader>);
static_assert(std::is_standard_layout_v<ClientInfoPdu> && std::is_trivial_v<ClientInfoPdu>);
This will fail to compile. Treating the types as POD (as you do) invokes
Undefined Behaviour.
This is likely the explanation for the fact that "it stops working" with some changes. It never worked: it might just accidentally have appeared to work, but it was undefined behaviour.
It's not easy to achieve POD-ness while still getting the convenience of the
constructors. In fact, I don't think that's possible. In short, if you want to
treat your structs as C-style POD types, make them... C-style POD types.
Another thing: a possible implementation of `PduHeader I
can see working for you looks a bit like so:
enum MsgId{CLIENT_INFO_PDU=0x123};
struct PduHeader {
MsgId id;
size_t payload_size;
size_t get_pdu_payload_size() const { return payload_size; }
};
Here, again you might have/need endianness conversions.
Suggestion
In short, if you want this to work, I'd say keep it simple.
Instead of creating non-POD types all over the place that are responsible for endianness conversion by adding getters/setters for each value, why not create a simple user-defined-type that does this always, and use them instead?
struct PduHeader {
Short id; // or e.g. uint8_t
Long payload_size;
};
struct ClientInfoPdu {
PduHeader pdu_header; // or inheritance, same effect
Short client_receiver_port;
};
Then just use it as a POD struct:
while (true) {
ClientInfoPdu client_info_pdu;
init_pdu(client_info_pdu);
auto n = socket.send_to(boost::asio::buffer(&client_info_pdu, sizeof(client_info_pdu)), remote_endpoint);
printf("sent %lu bytes\n", n);
std::this_thread::sleep_for(1s);
}
The function init_pdu can be implemented with overloads per submessage:
void init_pdu(ClientInfoPdu& msg) {
msg.pdu_header.id = CLIENT_INFO_PDU;
msg.pdu_header.payload_size = sizeof(msg);
}
There are variations on this where it can become a template or take a
PduHeder& (if your message inherits instead of aggregates). But the basic
principle is the same.
Endianness Conversion
Now you'll noticed I avoided using uint32_t/uint16_t directly (though uint8_t is fine because it doesn't need byte ordering). Instead, you could define Long and Short as simple POD wrappers around them:
struct Short {
operator uint16_t() const { return ntohs(value); }
Short& operator=(uint16_t v) { value = htons(v); return *this; }
private:
uint16_t value;
};
struct Long {
operator uint32_t() const { return ntohl(value); }
Long& operator=(uint32_t v) { value = htonl(v); return *this; }
private:
uint32_t value;
};
The assignment and conversions mean that you can use it as just another
int32_t/int16_t except that the necessary conversions are always done.
If you want to satnd on the shoulder of giants instead, you can use the better types from Boost Endian, which also has lots more advanced facilities
DEMO
Live On Coliru
#include <type_traits>
#include <cstdint>
#include <thread>
#include <arpa/inet.h>
using namespace std::chrono_literals;
#pragma pack(push, 1)
enum MsgId{CLIENT_INFO_PDU=0x123};
struct Short {
operator uint16_t() const { return ntohs(value); }
Short& operator=(uint16_t v) { value = htons(v); return *this; }
private:
uint16_t value;
};
struct Long {
operator uint32_t() const { return ntohl(value); }
Long& operator=(uint32_t v) { value = htonl(v); return *this; }
private:
uint32_t value;
};
static_assert(std::is_standard_layout_v<Short>);
static_assert(std::is_trivial_v<Short>);
static_assert(std::is_standard_layout_v<Long>);
static_assert(std::is_trivial_v<Long>);
struct PduHeader {
Short id; // or e.g. uint8_t
Long payload_size;
};
struct ClientInfoPdu {
PduHeader pdu_header; // or inheritance, same effect
Short client_receiver_port;
};
void init_pdu(ClientInfoPdu& msg) {
msg.pdu_header.id = CLIENT_INFO_PDU;
msg.pdu_header.payload_size = sizeof(msg);
}
static_assert(std::is_standard_layout_v<PduHeader> && std::is_trivial_v<PduHeader>);
static_assert(std::is_standard_layout_v<ClientInfoPdu> && std::is_trivial_v<ClientInfoPdu>);
#pragma pack(pop)
#include <boost/asio.hpp>
//#include <unistd.h>
using boost::asio::ip::udp;
#define SERVER_IP "127.0.0.1"
#define SERVER_PORT 6767
#define RECEIVE_PORT 6868
struct ServerConnectionThread {
void execute() {
boost::asio::io_service io_service;
udp::endpoint const remote_endpoint =
udp::endpoint(boost::asio::ip::address::from_string(SERVER_IP), SERVER_PORT);
udp::socket socket(io_service);
socket.open(udp::v4());
while (true) {
ClientInfoPdu client_info_pdu;
init_pdu(client_info_pdu);
auto n = socket.send_to(boost::asio::buffer(&client_info_pdu, sizeof(client_info_pdu)), remote_endpoint);
printf("sent %lu bytes\n", n);
std::this_thread::sleep_for(1s);
}
}
};
int main(){ }
I have a use case where one thread reads message into a large buffer and the distributes the processing to a bunch of threads. The buffer is shared by multiple threads after that. Its read-only and when the last thread finishes, the buffer has to be freed. The buffer is allocated from a lock-free slab allocator.
My initial design was to use shared_ptr for the buffer. But the buffer can be of different size. My way of getting around it was do something like this.
struct SharedBuffer {
SharedBuffer (uint16_t len, std::shared_ptr<void> ptr)
: _length(len), _buf(std::move(ptr))
{
}
uint8_t data () { return (uint8_t *)_buf.get(); }
uint16_t length
std::shared_ptr<void> _buf; // type-erase the shared_ptr as the SharedBuffer
// need to stored in some other structs
};
Now the allocator will allocate the shared_ptr like this:
SharedBuffer allocate (size_t size)
{
auto buf = std::allocate_shared<std::array<uint8_t, 16_K>>(myallocator);
return SharedBuffer{16_K, buf}; // type erase the std::array
}
And the SharedBuffer is enqueued to each thread who wants it.
Now I think, I am doing lot of stuff unnecessarily, I can sort of make do with boost::intrusive_ptr with the below scheme. Things are bit C'ish- as I am using variable size array. Here I have changed the slab allocator with a operator new() for the sake of simplicity. I wanted to run it by to see if this implementation is okay.
template <typename T>
inline int atomicIncrement (T* t)
{
return __atomic_add_fetch(&t->_ref, 1, __ATOMIC_ACQUIRE);
}
template <typename T>
inline int atomicDecrement (T* t)
{
return __atomic_sub_fetch(&t->_ref, 1, __ATOMIC_RELEASE);
}
class SharedBuffer {
public:
friend int atomicIncrement<SharedBuffer>(SharedBuffer*);
friend int atomicDecrement<SharedBuffer>(SharedBuffer*);
SharedBuffer(uint16_t len) : _length(len) {}
uint8_t *data ()
{
return &_data[0];
}
uint16_t length () const
{
return _length;
}
private:
int _ref{0};
const uint16_t _length;
uint8_t _data[];
};
using SharedBufferPtr = boost::intrusive_ptr<SharedBuffer>;
SharedBufferPtr allocate (size_t size)
{
// dummy implementation
void *p = ::operator new (size + sizeof(SharedBuffer));
// I am not explicitly constructing the array of uint8_t
return new (p) SharedBuffer(size);
}
void deallocate (SharedBuffer* sbuf)
{
sbuf->~SharedBuffer();
// dummy implementation
::operator delete ((void *)sbuf);
}
void intrusive_ptr_add_ref(SharedBuffer* sbuf)
{
atomicIncrement(sbuf);
}
void intrusive_ptr_release (SharedBuffer* sbuf)
{
if (atomicDecrement(sbuf) == 0) {
deallocate(sbuf);
}
}
I'd use the simpler implementation (using shared_ptr) unless you are avoiding specific problems (i.e. profile first).
Side Note: you can use boost::shared_pointer<> with boost::make_shared<T[]>(N), which is being [added to the standard library in c++20.
Note that allocate_shared already embeds the control block into the same allocation like you do with the intrusive approach.
Finally, I'd use std::atomic_int so you have a clear contract that cannot (accidentally) be used wrong. At the same time, it'll remove the remaining bit of complexity.
Is there a way to write a copy-constructor for a class (say, Copyable, that holds a std::unique_ptr to a Base class (but really is storing Derived objects.
A quick test shows the expected slicing occurs, because Copyable doesn't know the real type it's holding. So I suppose a clone method is needed, but I'm wondering if there is a way to let the compiler handle this in some better way?
The slicing code:
#include <algorithm>
#include <iostream>
#include <memory>
struct Base
{
Base(int i = 0) : i(i) {}
virtual ~Base() = default;
int i;
virtual int f() { return i; }
};
struct Derived : Base
{
Derived() = default;
virtual int f() override { return 42; }
};
struct Copyable
{
Copyable(std::unique_ptr<Base>&& base) : data(std::move(base)) {}
Copyable(const Copyable& other)
{
data = std::make_unique<Base>(*other.data);
}
std::unique_ptr<Base> data;
};
int main()
{
Copyable c(std::make_unique<Derived>());
Copyable c_copy = c;
std::cout << c_copy.data->f() << '\n';
}
The clone code:
#include <algorithm>
#include <iostream>
#include <memory>
struct Base
{
Base(int i = 0) : i(i) {}
virtual ~Base() = default;
int i;
virtual int f() { return i; }
virtual Base* clone() { return new Base(i); }
};
struct Derived : Base
{
Derived() = default;
virtual int f() override { return 42; }
virtual Derived* clone() override { return new Derived(); }
};
struct Copyable
{
Copyable(std::unique_ptr<Base>&& base) : data(std::move(base)) {}
Copyable(const Copyable& other)
{
data.reset(other.data->clone());
}
std::unique_ptr<Base> data;
};
int main()
{
Copyable c(std::make_unique<Derived>());
Copyable c_copy = c;
std::cout << c_copy.data->f() << '\n';
}
Obviously the clone code works. Thing is, there's some things in it I'd like to avoid:
raw new.
a random function that needs to be part of the interface.
This function returns a raw pointer.
Every user of this class that wants to be copyable needs to call this function.
So, is there a "clean" alternative?
Note I want to use smart pointers for all the obvious reasons, I just need a deep copying std::unique_ptr. Something like std::copyable_unique_ptr, combining optional move semantics with a deep copying copy constructor. Is this the cleanest way? Or does that only add the the confusion?
You can certainly create a clone_ptr-class for any object you know statically how to clone.
It would hold a pointer to the object, and a pointer to a function for cloning said object, probably from converting a stateless lambda.
i trying to develop a Finite State Machine with template meta programming techniques, but i getting stuck with a map that it has to be fill at compile time, this the code(gcc 4.8 c++11):
#include <functional>
#include <type_traits>
#include <iostream>
#include <unordered_map>
namespace NSStateMachine {
//Definicion de estado unidad
template<class FSM, class From, class Event, class TO, bool(FSM::* transicion_fn)(const Event &)>
struct Transition
{
using FSM_TYPE=FSM;
using FROM_STATE= From;
using EVENT_TYPE= Event;
using TO_STATE_TYPE=TO;
using EVENT_BASE_TYPE = typename Event::BASE_TYPE;
static bool do_transition(FSM_TYPE& currenState, EVENT_BASE_TYPE const& aEvent)
{
return (currenState.*transicion_fn)(static_cast<EVENT_TYPE const&>(aEvent));
}
};
//States
template<class Transition, const Transition * const TransitionPtr, class ... Args>
class StateMachine
{
public:
StateMachine():transitionMap{{static_cast<typename Transition::TransitionID>(*TransitionPtr::TransitionID_Value),nullptr}}
{}
template<class Event>
bool evalEvent(const Event & aEvent)
{
std::cout<<"evento recibido"<<std::endl;
}
std::unordered_map<typename Transition::TransitionID, const Transition * const > transitionMap ;
};
}
int main()
{
//it doesnt compile, i canoot create the state machine
return 0;
}
The compile error:
error: 'TransitionPtr' is not a class, namespace, or enumeration
StateMachine():transitionMap{{static_cast<typename Transition::TransitionID>(*TransitionPtr::TransitionID_Value),nullptr}}
^
The problem seem to be in the line
transitionMap{{static_cast<typename Transition::TransitionID>(*TransitionPtr::TransitionID_Value),nullptr}}
i will try to init the unorderer_map with the automatic constructor.
i have defined this Transition::TransitionID as a class variable defined in the class represented by the template argument
I will really appreciate any help.
Thx!!!!
i have already test with default types , it compile and work this
The error message is pretty clear. TransitionPtr is a pointer, not a type, so you can't use it to the left of :: in TransitionPtr::TransitionID_Value.
Also, I don't think you'll find a way to initialize an unordered_set at compile time, since it doesn't have constexpr constructors and in general almost certainly uses heap allocations.
I want to replace some code that uses boost::interprocess shared memory. One advantage of shared memory is that you can impose limits on the maximum amount of memory it can use. I'm looking for a custom allocator, based off std::allocator that can do this.
Only particular classes in the program will use this allocator, everything else uses the defaulted std::allocator and are only limited by available RAM.
I'm trying to write one of my own but I'm running into issues, mainly with how to share state among the allocator copies that are created by STL containers. State includes the number of free bytes remaining and the maximum size the allocator can use. I thought I could get away with making them thread_local but then several different instances of the same class will all allocate and deallocate from the same limited heap, which is not what I want. I'm beginning to think it's not possible, hence this question here. Neither contiguous allocation nor performance are major requirements for now.
The hard limit on the memory size cannot be a template parameter either, it's read from a config file.
Edit: The issue with sharing state is that some containers call the default constructor of the allocator type. Obviously this constructor cannot easily know anything about the outside world even if shared_ptr is used it will be nullptr initialised. For example, look at the source code for std::string::clear
g++ (Ubuntu 4.8.4-2ubuntu1~14.04) 4.8.4
After following the hints above I came up with this which seems to work ok for POD types, but things fall apart when I try to make a Vector or Map that uses String:
#include <string>
#include <vector>
#include <map>
#include <atomic>
#include <memory>
struct SharedState
{
SharedState()
: m_maxSize(0),
m_bytesRemaining(0)
{
}
SharedState(std::size_t maxSize)
: m_maxSize(maxSize),
m_bytesRemaining(maxSize)
{
}
void allocate(std::size_t bytes) const {
if (m_bytesRemaining < bytes) {
throw std::bad_alloc();
}
m_bytesRemaining -= bytes;
}
void deallocate(std::size_t bytes) const {
m_bytesRemaining += bytes;
}
std::size_t getBytesRemaining() const {
return m_bytesRemaining;
}
const std::size_t m_maxSize;
mutable std::atomic<std::size_t> m_bytesRemaining;
};
// --------------------------------------
template <typename T>
class BaseLimitedAllocator : public std::allocator<T>
{
public:
using size_type = std::size_t;
using pointer = T*;
using const_pointer = const T*;
using propagate_on_container_move_assignment = std::true_type;
template <typename U>
struct rebind
{
typedef BaseLimitedAllocator<U> other;
};
BaseLimitedAllocator() noexcept = default;
BaseLimitedAllocator(std::size_t maxSize) noexcept
: m_state(new SharedState(maxSize)) {
}
BaseLimitedAllocator(const BaseLimitedAllocator& other) noexcept {
m_state = other.m_state;
}
template <typename U>
BaseLimitedAllocator(const BaseLimitedAllocator<U>& other) noexcept {
m_state = other.m_state;
}
pointer allocate(size_type n, const void* hint = nullptr) {
m_state->allocate(n * sizeof(T));
return std::allocator<T>::allocate(n, hint);
}
void deallocate(pointer p, size_type n) {
std::allocator<T>::deallocate(p, n);
m_state->deallocate(n * sizeof(T));
}
public:
std::shared_ptr<SharedState> m_state; // This must be public for the rebind copy constructor.
};
template <typename T, typename U>
inline bool operator==(const BaseLimitedAllocator<T>&, const BaseLimitedAllocator<U>&) {
return true;
}
template <typename T, typename U>
inline bool operator!=(const BaseLimitedAllocator<T>&, const BaseLimitedAllocator<U>&) {
return false;
}
struct LimitedAllocator : public BaseLimitedAllocator<char>
{
LimitedAllocator(std::size_t maxSize)
: BaseLimitedAllocator<char>(maxSize) {
}
template <typename U>
using Other = typename BaseLimitedAllocator<char>::template rebind<U>::other;
};
// -----------------------------------------
// Example usage:
class SomeClass
{
public:
using String = std::basic_string<char, std::char_traits<char>, LimitedAllocator::Other<char>>;
template <typename T>
using Vector = std::vector<T, LimitedAllocator::Other<T>>;
template <typename K, typename V>
using Map = std::map<K, V, std::less<K>, LimitedAllocator::Other<std::pair<const K, V>>>;
Complex()
: allocator(256),
s(allocator),
v(allocator),
m(std::less<int>(), allocator) // Cannot only specify the allocator. Annoying.
{
}
const LimitedAllocator allocator;
String s;
Vector<int> v;
Map<int, String> m;
};