b2database.H

//------------------------------------------------------------------------
// b2database.H --
//
//
// written by Mathias Doreille
//
// Copyright (c) 2004-2015,2016,2017
//     SMR Engineering & Development SA
//     2502 Bienne, Switzerland
//
// All Rights Reserved.  Proprietary source code.  The contents of
// this file may not be disclosed to third parties, copied or
// duplicated in any form, in whole or in part, without the prior
// written permission of SMR.
//------------------------------------------------------------------------
 
#ifndef __B2DATABASE_H__
#define __B2DATABASE_H__
 
#include <fnmatch.h>
#include <memcom.h>
 
#include <complex>
#include <map>
#include <sstream>
#include <string>
#include <vector>
 
#include "b2ppconfig.h"
#include "utils/b2linear_algebra.H"
#include "utils/b2logging.H"
#include "utils/b2object.H"
#include "utils/b2rtable.H"
#include "utils/b2time.H"
 
extern "C" int _mcDBopenFile(
      int handle, const char* filename, const char* ftype, mcSize npage, mcSize lpage);
 
inline std::string mcErrGetMsgAndReset(int code) {
    const char* err_msg = mcErrGetMsg(code);
    std::string res = err_msg ? err_msg : "Invalid MemCom error code";
    mcErrReset();
    return res;
}
 
namespace b2000 {
 
class SetName : public std::string {
public:
    SetName() {}
 
    SetName(const SetName& name) : std::string(name) {}
 
    operator const char*() const { return c_str(); }
    template <typename T1>
    SetName(const T1& a1) {
        std::ostringstream o;
        o << a1;
        std::string::operator=(o.str());
    }
    template <typename T1, typename T2>
    SetName(const T1& a1, const T2& a2) {
        std::ostringstream o;
        o << a1 << "." << a2;
        std::string::operator=(o.str());
    }
    template <typename T1, typename T2, typename T3>
    SetName(const T1& a1, const T2& a2, const T3& a3) {
        std::ostringstream o;
        o << a1 << "." << a2 << "." << a3;
        std::string::operator=(o.str());
    }
    template <typename T1, typename T2, typename T3, typename T4>
    SetName(const T1& a1, const T2& a2, const T3& a3, const T4& a4) {
        std::ostringstream o;
        o << a1 << "." << a2 << "." << a3 << "." << a4;
        std::string::operator=(o.str());
    }
    template <typename T1, typename T2, typename T3, typename T4, typename T5>
    SetName(const T1& a1, const T2& a2, const T3& a3, const T4& a4, const T5& a5) {
        std::ostringstream o;
        o << a1 << "." << a2 << "." << a3 << "." << a4 << "." << a5;
        std::string::operator=(o.str());
    }
 
    SetName& gname(const std::string& gname_) {
        replace_field(0, gname_);
        return *this;
    }
 
    template <typename T>
    SetName& gname(const T& gname_) {
        std::ostringstream o;
        o << gname_;
        return gname(o.str());
    }
 
    std::string get_gname() const { return substr_field(0); }
 
    bool gname_undef() const { return is_undef(0); }
 
    SetName& branch(const std::string& branch_) {
        replace_field(1, branch_);
        return *this;
    }
 
    template <typename T>
    SetName& branch(const T& branch_) {
        std::ostringstream o;
        o << branch_;
        return branch(o.str());
    }
 
    std::string get_branch() const { return substr_field(1); }
 
    bool branch_undef() const { return is_undef(1); }
 
    SetName& cycle(const std::string& cycle_) {
        replace_field(2, cycle_);
        return *this;
    }
 
    template <typename T>
    SetName& cycle(const T& cycle_) {
        std::ostringstream o;
        o << cycle_;
        return cycle(o.str());
    }
 
    std::string get_cycle() const { return substr_field(2); }
 
    bool cycle_undef() const { return is_undef(2); }
 
    SetName& subcycle(const std::string& subcycle_) {
        replace_field(3, subcycle_);
        return *this;
    }
 
    template <typename T>
    SetName& subcycle(const T& subcycle_) {
        std::ostringstream o;
        o << subcycle_;
        return subcycle(o.str());
    }
 
    std::string get_subcycle() const { return substr_field(3); }
 
    bool subcycle_undef() const { return is_undef(3); }
 
    SetName& case_(const std::string& case__) {
        replace_field(4, case__);
        return *this;
    }
 
    template <typename T>
    SetName& case_(const T& case__) {
        std::ostringstream o;
        o << case__;
        return case_(o.str());
    }
 
    std::string get_case() const { return substr_field(4); }
 
    bool case_undef() const { return is_undef(4); }
 
    SetName& subcase(const std::string& subcase_) {
        replace_field(5, subcase_);
        return *this;
    }
 
    template <typename T>
    SetName& subcase(const T& subcase_) {
        std::ostringstream o;
        o << subcase_;
        return subcase(o.str());
    }
 
    std::string get_subcase() const { return substr_field(5); }
 
    bool subcase_undef() const { return is_undef(5); }
 
    bool match(const std::string& pattern) const {
        return (fnmatch(pattern.c_str(), c_str(), 0) == 0);
    }
 
private:
    void replace_field(int pos, const std::string& s) {
        const int pos_save = pos;
        int begin_ = 0;
        while (pos--) {
            if ((begin_ = std::string::find(".", begin_)) == -1) {
                if (pos) {
                    *this += ".0";
                    begin_ = std::string::size() - 1;
                } else {
                    *this += ".";
                    begin_ = std::string::size();
                }
            } else {
                ++begin_;
            }
        }
        int end_ = std::string::find(".", begin_);
        std::string::replace(begin_, (end_ == -1 ? std::string::size() : end_) - begin_, s);
 
        // Special rule for subcase/mode.
        if (pos_save == 5 && s == "0") { resize(begin_ - 1); }
    }
 
    std::string substr_field(int pos) const {
        int begin_ = 0;
        while (pos--) {
            if ((begin_ = std::string::find(".", begin_)) == -1) { return ""; }
            ++begin_;
        }
        int end_ = std::string::find(".", begin_);
        return std::string::substr(begin_, (end_ == -1 ? std::string::size() : end_) - begin_);
    }
 
    bool is_undef(int pos) const {
        std::string tmp = substr_field(pos);
        return tmp.empty() || tmp == "0" || tmp == "*";
    }
};
 
template <typename T>
struct MemcomTypeName {
    static const char* name;
};
 
template <typename T>
const char* MemcomTypeName<T>::name = "U";
 
template <>
const char* MemcomTypeName<mcInt32>::name;
 
template <>
const char* MemcomTypeName<unsigned int>::name;
 
template <>
const char* MemcomTypeName<mcInt64>::name;
 
template <>
const char* MemcomTypeName<mcFloat64>::name;
 
template <>
const char* MemcomTypeName<mcFloat32>::name;
 
template <>
const char* MemcomTypeName<b2000::csda<double> >::name;
 
typedef std::vector<RTable> ArrayRTable;
 
class DBSlice {
public:
    DBSlice(mcOff start_, mcOff end_ = 0, mcOff step_ = 1)
        : start(start_), end(end_ == 0 ? start_ + 1 : end_), step(step_) {}
 
    size_t size() const { return (end - start) / step; }
 
    bool is_null() const { return this == &null; }
 
    bool is_all() const { return this == &all; }
 
    void set_mcslice(mcSlice* sl) const {
        sl->start = start + 1;
        sl->end = end;
        sl->start = step;
    }
 
    static DBSlice null;
    static DBSlice all;
 
private:
    friend class DataBase;
    mcOff start;
    mcOff end;
    mcOff step;
};
 
class DataBase {
public:
    DataBase()
        : handler(-1),
          is_readonly(false),
          close_db_on_delete(true),
          logger(logging::get_logger("database.b2000db")),
          time_sync(0) {}
 
    virtual ~DataBase();
 
    void init(const std::string& name);
 
    std::string get_name() const { return name; }
 
    std::string get_logging_directory();
 
    void open(bool readonly = false);
 
    virtual void close() {
        if (handler != -1) {
            int status = mcDBcloseFile(handler);
            if (status < 0) { IOError() << mcErrGetMsgAndReset(status) << THROW; }
            handler = -1;
        }
    }
 
    int get_handle() { return handler; }
 
    bool has_key(const std::string& key) const {
        return (mcDBinqSetAtt(handler, key.c_str(), nullptr) == 1);
    }
 
    typedef std::set<SetName> SetNameSet;
 
    void add_keys(const std::string& pattern, SetNameSet& set_names) const {
        mcSetAttributes att;
        for (int iter = 0; mcDBgetSetNextIter(handler, &iter, &att);) {
            SetName set_name(att.name);
            if (set_name.match(pattern)) { set_names.insert(set_name); }
        }
    }
 
    virtual void sync() {
        Time t1;
        int status = mcDBsaveFileNoSync(handler);
        if (status < 0) { IOError() << mcErrGetMsgAndReset(status) << THROW; }
        Time t2;
        time_sync = double(t2 - t1) * 10;
        last_sync = t1;
    }
 
    void sync_delayed() {
        if (double(Time() - last_sync) > time_sync) { sync(); }
    }
 
    virtual void temporary_close() {
        int status = mcDBcloseFile(handler);
        if (status < 0) { IOError() << mcErrGetMsgAndReset(status) << THROW; }
    }
 
    virtual void reopen() {
        std::string arch_path = path::join(db_name, "archives.mc");
        if (_mcDBopenFile(handler, arch_path.c_str(), (is_readonly ? "o" : "rw"), 0, 0)
            != handler) {
            IOError() << "Cannot reopen the B2000++ database \"" << db_name << "\"." << THROW;
        }
    }
 
    class TemporaryClose {
    public:
        TemporaryClose(DataBase& db_) : db(db_) { db.temporary_close(); }
        ~TemporaryClose() { db.reopen(); }
 
    private:
        DataBase& db;
    };
 
    size_t ndim(const std::string& key) const {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "The dataset " << key << " does not exist." << THROW;
        }
        return att.ndim;
    }
 
    const std::vector<size_t> dims(const std::string& key) const {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "The dataset " << key << " does not exist." << THROW;
        }
        return std::vector<size_t>(att.dim, att.dim + att.ndim);
    }
 
    const std::type_info& element_type(const std::string& key) const {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "The dataset " << key << " does not exist." << THROW;
        }
        switch (att.type[0]) {
            case 'I':
                return typeid(mcInt32);
            case 'J':
                return typeid(mcInt64);
            case 'E':
                return typeid(mcFloat32);
            case 'F':
                return typeid(mcFloat64);
            case 'C':
                return typeid(std::complex<mcFloat64>);
            case 'Z':
                return typeid(std::complex<mcFloat64>);
            case 'K':
                return typeid(std::string);
            case '$':
                return typeid(RTable);
            default:
                if (att.type[0] == 'S' && att.type[0] == 'T') { return typeid(ArrayRTable); }
                DBError() << "The memcom type " << att.type << " is not supported by B2000++"
                          << THROW;
                break;
        }
        return typeid(void);
    }
 
    void remove(const std::string& key) {
        int status = mcDBdeleteSet(handler, key.c_str());
        if (status) {
            DBError() << "mcDBdeleteSet returns an unexpected error: "
                      << mcErrGetMsgAndReset(status) << "." << THROW;
        }
    }
 
    void rename(const std::string& oldkey, const std::string& newkey) {
        if (has_key(newkey) && has_key(oldkey)) { remove(newkey); }
        int status = mcDBrenameSet(handler, oldkey.c_str(), newkey.c_str());
        if (status) {
            DBError() << "mcDBrenameSet returns an unexpected error: "
                      << mcErrGetMsgAndReset(status) << "." << THROW;
        }
    }
 
    mcSetAttributes get_attributes(const std::string& key) const {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "The dataset " << key << " does not exist." << THROW;
        }
        return att;
    }
 
    template <typename T>
    void get(const std::string& key, std::vector<T>& v, const DBSlice& s1 = DBSlice::all) const {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "Cannot read dataset " << key << ", it does not exist." << THROW;
        }
        if (att.ndim != 1 && att.dim[1] != 1) {
            DBError() << "Cannot read dataset " << key << ", it has " << att.ndim
                      << " instead of 1 dimensions." << THROW;
        }
        bool convert = false;
        if (MemcomTypeName<T>::name[0] != att.type[0]) {
            if (!can_convert_data(att.type, MemcomTypeName<T>::name)) {
                DBError() << "Cannot read dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name << "." << THROW;
            }
            convert = true;
        }
 
        if (v.size() == 0) {
            if (s1.is_all()) {
                v.resize(att.dim[0]);
            } else {
                v.resize(s1.size());
            }
        } else if (s1.is_all()) {
            if (mcOff(v.size()) != att.dim[0]) {
                DBError() << "Cannot read dataset " << key << ", its first dimension is "
                          << att.dim[0] << " instead of " << v.size() << "." << THROW;
            }
        } else if (v.size() != s1.size()) {
            DBError() << "Cannot read dataset " << key << ", the size of the vector is " << v.size()
                      << " but the size of the DBSlice is " << s1.size() << "." << THROW;
        }
 
        void* p = (void*)&v[0];
        std::vector<char> tmp;
        if (convert) {
            tmp.resize(mcSizeof(att.type) * v.size());
            p = (void*)&tmp[0];
        }
 
        if (s1.is_all()) {
            int status = mcDBgetSetArray(handler, key.c_str(), v.size(), p);
            if (status != 0) {
                DBError() << "mcDBgetSetposArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl;
            s1.set_mcslice(&sl);
            int status = mcDBgetSetsliceArray(handler, key.c_str(), 1, &sl, 0, nullptr, p);
            if (status != 0) {
                DBError() << "mcDBgetSetsliceArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
 
        if (convert) { convert_data(v.size(), att.type, MemcomTypeName<T>::name, &tmp[0], &v[0]); }
    }
 
    template <typename T>
    void get(const std::string& key, std::vector<T>& v, const DBSlice& s1, const DBSlice& s2) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "Cannot read dataset " << key << ", it does not exist." << THROW;
        }
        if (att.ndim != 2) {
            DBError() << "Cannot read dataset " << key << ", it has " << att.ndim
                      << " instead of 2 dimensions." << THROW;
        }
        bool convert = false;
        if (MemcomTypeName<T>::name[0] != att.type[0]) {
            if (!can_convert_data(att.type, MemcomTypeName<T>::name)) {
                DBError() << "Cannot read dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name << "." << THROW;
            }
            convert = true;
        }
        if (v.size() == 0) {
            if (s1.is_all()) {
                v.resize(att.dim[0]);
            } else {
                v.resize(s1.size());
            }
        } else if (s1.is_all()) {
            if (mcOff(v.size()) != att.dim[0]) {
                DBError() << "Cannot read dataset " << key << ", its first dimension is "
                          << att.dim[0] << " instead of " << v.size() << "." << THROW;
            }
        } else if (v.size() != s1.size()) {
            DBError() << "Cannot read dataset " << key << ", the size of the vector is " << v.size()
                      << " but the size of the DBSlice is " << s1.size() << "." << THROW;
        }
        if (s2.is_all() || s2.size() != 1) {
            DBError() << "The DBSlice " << key << " has not dimension 1." << THROW;
        }
 
        void* p = (void*)&v[0];
        std::vector<char> tmp;
        if (convert) {
            tmp.resize(mcSizeof(att.type) * v.size());
            p = (void*)&tmp[0];
        }
 
        if (s1.is_all()) {
            int status =
                  mcDBgetSetposArray(handler, key.c_str(), att.dim[0] * s2.start + 1, v.size(), p);
            if (status != 0) {
                DBError() << "mcDBgetSetposArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl[2];
            s2.set_mcslice(sl);
            s1.set_mcslice(sl + 1);
            int status = mcDBgetSetsliceArray(handler, key.c_str(), 1, sl, 0, nullptr, p);
            if (status != 0) {
                DBError() << "mcDBgetSetsliceArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
 
        if (convert) { convert_data(v.size(), att.type, MemcomTypeName<T>::name, &tmp[0], &v[0]); }
    }
 
    template <typename T>
    void get(
          const std::string& key, b2linalg::Vector<T, b2linalg::Vincrement_ref> v,
          const DBSlice& s1 = DBSlice::all) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "Cannot read dataset " << key << ", it does not exist." << THROW;
        }
        if (att.ndim != 1) {
            DBError() << "Cannot read dataset " << key << ", it has " << att.ndim
                      << " instead of 1 dimensions." << THROW;
        }
        bool convert = false;
        if (MemcomTypeName<T>::name[0] != att.type[0]) {
            if (!can_convert_data(att.type, MemcomTypeName<T>::name)) {
                DBError() << "Cannot read dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name << "." << THROW;
            }
            convert = true;
        }
 
        if (s1.is_all()) {
            if (mcOff(v.size()) != att.dim[0]) {
                DBError() << "Cannot read dataset " << key << " its first dimension is "
                          << att.dim[0] << " instead of " << v.size() << "." << THROW;
            }
        } else if (v.size() != s1.size()) {
            DBError() << "Cannot read dataset " << key << ", the size of the vector is " << v.size()
                      << " but the size of the DBSlice is " << s1.size() << "." << THROW;
        }
 
        void* p = (void*)&v[0];
        std::vector<char> tmp;
        if (convert) {
            tmp.resize(mcSizeof(att.type) * v.size());
            p = (void*)&tmp[0];
        }
 
        if (s1.is_all()) {
            int status = mcDBgetSetArray(handler, key.c_str(), v.size(), p);
            if (status != 0) {
                DBError() << "mcDBgetSetposArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl;
            s1.set_mcslice(&sl);
            int status = mcDBgetSetsliceArray(handler, key.c_str(), 1, &sl, 0, nullptr, p);
            if (status != 0) {
                DBError() << "mcDBgetSetsliceArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
 
        if (convert) { convert_data(v.size(), att.type, MemcomTypeName<T>::name, &tmp[0], &v[0]); }
    }
 
    template <typename T>
    void get(
          const std::string& key, b2linalg::Vector<T, b2linalg::Vincrement_ref> v,
          const DBSlice& s1, const DBSlice& s2) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "Cannot read dataset " << key << ", it does not exist." << THROW;
        }
        if (att.ndim != 2) {
            DBError() << "Cannot read dataset " << key << ", it has " << att.ndim
                      << " instead of 2 dimensions." << THROW;
        }
        bool convert = false;
        if (MemcomTypeName<T>::name[0] != att.type[0]) {
            if (!can_convert_data(att.type, MemcomTypeName<T>::name)) {
                DBError() << "Cannot read dataset " << key << " its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name << "." << THROW;
            }
            convert = true;
        }
 
        if (s1.is_all()) {
            if (mcOff(v.size()) != att.dim[0]) {
                DBError() << "Cannot read dataset " << key << " its first dimension is "
                          << att.dim[0] << " instead of " << v.size() << "." << THROW;
            }
        } else if (v.size() != s1.size()) {
            DBError() << "Cannot read dataset " << key << ", the size of the vector is " << v.size()
                      << " but the size of the DBSlice is " << s1.size() << "." << THROW;
        }
        if (s2.is_all() || s2.size() != 1) {
            DBError() << "Cannot read dataset " << key << ", the DBSlice " << key
                      << " has not dimension 1." << THROW;
        }
 
        void* p = (void*)&v[0];
        std::vector<char> tmp;
        if (convert) {
            tmp.resize(mcSizeof(att.type) * v.size());
            p = (void*)&tmp[0];
        }
 
        if (s1.is_all()) {
            int status =
                  mcDBgetSetposArray(handler, key.c_str(), att.dim[0] * s2.start + 1, v.size(), p);
            if (status != 0) {
                DBError() << "mcDBgetSetposArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl[2];
            s2.set_mcslice(sl);
            s1.set_mcslice(sl + 1);
            int status = mcDBgetSetsliceArray(handler, key.c_str(), 1, sl, 0, nullptr, p);
            if (status != 0) {
                DBError() << "mcDBgetSetsliceArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
 
        if (convert) { convert_data(v.size(), att.type, MemcomTypeName<T>::name, &tmp[0], &v[0]); }
    }
 
    template <typename T>
    void get(
          const std::string& key, b2linalg::Matrix<T, b2linalg::Mrectangle>& m,
          const DBSlice& s1 = DBSlice::all, const DBSlice& s2 = DBSlice::all) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "Cannot read dataset " << key << ", it does not exist." << THROW;
        }
        if (att.ndim != 2) {
            DBError() << "Cannot read dataset " << key << ", it has " << att.ndim
                      << " instead of 2 dimensions." << THROW;
        }
        bool convert = false;
        if (MemcomTypeName<T>::name[0] != att.type[0]) {
            if (!can_convert_data(att.type, MemcomTypeName<T>::name)) {
                DBError() << "Cannot read dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name << "." << THROW;
            }
            convert = true;
        }
 
        if (m.size1() != 0) {
            if (s1.is_all()) {
                if (mcOff(m.size1()) != att.dim[1]) {
                    DBError() << "Cannot read dataset set " << key << ", its first dimension is "
                              << att.dim[1] << " instead of " << m.size1() << "." << THROW;
                }
            } else if (m.size1() != s1.size()) {
                DBError() << "Cannot read dataset " << key
                          << ", the first dimension of the matrix is " << m.size1()
                          << " but the size of the first DBSlice is " << s1.size() << "." << THROW;
            }
        }
        if (m.size2() != 0) {
            if (s2.is_all()) {
                if (mcOff(m.size2()) != att.dim[0]) {
                    DBError() << "Cannot read dataset " << key << ", its second dimension is "
                              << att.dim[0] << " instead of " << m.size2() << "." << THROW;
                }
            } else if (m.size2() != s2.size()) {
                DBError() << "Cannot read dataset " << key
                          << ", the second dimension of the matrix is " << m.size2()
                          << " but the size of the second DBSlice is " << s2.size() << "." << THROW;
            }
        }
 
        if (s1.is_all() && s2.is_all()) {
            m.resize(att.dim[1], att.dim[0]);
        } else {
            m.resize(s1.is_all() ? att.dim[1] : s1.size(), s2.is_all() ? att.dim[0] : s2.size());
        }
 
        void* p = (void*)&m(0, 0);
        std::vector<char> tmp;
        if (convert) {
            tmp.resize(mcSizeof(att.type) * m.size1() * m.size2());
            p = (void*)&tmp[0];
        }
 
        if (s1.is_all() && s2.is_all()) {
            int status = mcDBgetSetArray(handler, key.c_str(), m.size1() * m.size2(), p);
            if (status != 0) {
                DBError() << "mcDBgetSetposArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl[2];
            if (s1.is_all()) {
                sl[0].start = 1;
                sl[0].end = att.dim[0];
                sl[0].step = 1;
            } else {
                s2.set_mcslice(sl);
            }
            if (s2.is_all()) {
                sl[1].start = 1;
                sl[1].end = att.dim[1];
                sl[1].step = 1;
            } else {
                s1.set_mcslice(sl + 1);
            }
            int status = mcDBgetSetsliceArray(handler, key.c_str(), 1, sl, 0, nullptr, p);
            if (status != 0) {
                DBError() << "mcDBgetSetsliceArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
 
        if (convert) {
            convert_data(
                  m.size1() * m.size2(), att.type, MemcomTypeName<T>::name, &tmp[0], &m(0, 0));
        }
    }
 
    template <typename T>
    void get(
          const std::string& key, b2linalg::Matrix<T, b2linalg::Mrectangle_increment_ref> m,
          const DBSlice& s1 = DBSlice::all, const DBSlice& s2 = DBSlice::all) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            DBError() << "Cannot read dataset " << key << ", it does not exist." << THROW;
        }
        if (att.ndim != 2) {
            DBError() << "Cannot read dataset " << key << ", it has " << att.ndim
                      << " instead of 2 dimensions." << THROW;
        }
        bool convert = false;
        if (MemcomTypeName<T>::name[0] != att.type[0]) {
            if (!can_convert_data(att.type, MemcomTypeName<T>::name)) {
                DBError() << "Cannot read dataset " << key << " its type is" << att.type
                          << " instead of " << MemcomTypeName<T>::name << "." << THROW;
            }
            convert = true;
        }
 
        if (s1.is_all()) {
            if (mcOff(m.size1()) != att.dim[1]) {
                DBError() << "Cannot read dataset " << key << " its first dimension is "
                          << att.dim[1] << " instead of " << m.size1() << "." << THROW;
            }
        } else if (m.size1() != s1.size()) {
            DBError() << "Cannot read dataset " << key << ", the first dimension of the matrix is "
                      << m.size1() << " but the size of the first DBSlice is " << s1.size() << "."
                      << THROW;
        }
        if (s2.is_all()) {
            if (mcOff(m.size2()) != att.dim[0]) {
                DBError() << "Cannot read dataset " << key << ", its second dimension is "
                          << att.dim[0] << " instead of " << m.size2() << "." << THROW;
            }
        } else if (m.size2() != s2.size()) {
            DBError() << "Cannot read dataset " << key << ", the second dimension of the matrix is "
                      << m.size2() << " but the size of the second DBSlice is " << s2.size() << "."
                      << THROW;
        }
 
        void* p = (void*)&m(0, 0);
        std::vector<char> tmp;
        if (convert) {
            tmp.resize(mcSizeof(att.type) * m.size1() * m.size2());
            p = (void*)&tmp[0];
        }
 
        if (s1.is_all() && s2.is_all()) {
            int status = mcDBgetSetArray(handler, key.c_str(), m.size1() * m.size2(), p);
            if (status != 0) {
                DBError() << "mcDBgetSetposArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl[2];
            if (s1.is_all()) {
                sl[0].start = 1;
                sl[0].end = att.dim[0];
                sl[0].step = 1;
            } else {
                s2.set_mcslice(sl);
            }
            if (s2.is_all()) {
                sl[1].start = 1;
                sl[1].end = att.dim[1];
                sl[1].step = 1;
            } else {
                s1.set_mcslice(sl + 1);
            }
            int status = mcDBgetSetsliceArray(handler, key.c_str(), 1, sl, 0, nullptr, p);
            if (status != 0) {
                DBError() << "mcDBgetSetsliceArray returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
 
        if (convert) {
            convert_data(
                  m.size1() * m.size2(), att.type, MemcomTypeName<T>::name, &tmp[0], &m(0, 0));
        }
    }
 
    template <typename T>
    void set(
          const std::string& key, const std::vector<T>& v, const DBSlice& s1 = DBSlice::all,
          bool new_set = false) {
        set(key, b2linalg::Vector<T, b2linalg::Vincrement_constref>(v), s1, new_set);
    }
 
    template <typename T>
    void set(
          const std::string& key, const b2linalg::Vector<T, b2linalg::Vincrement_constref>& v,
          const DBSlice& s1 = DBSlice::all, bool new_set = false) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            if (!s1.is_all()) {
                DBError() << "Cannot write dataset " << key
                          << " an existing dataset is requested, but the dataset "
                             "does not exist."
                          << THROW;
            }
            mcOff dim = v.size();
            int status = mcDBcreateSet(handler, key.c_str(), MemcomTypeName<T>::name, 1, &dim);
            if (status != 0) {
                DBError() << "mcDBcreateSet returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            if (new_set) {
                DBError() << "Cannot write dataset " << key
                          << ", a new dataset is requested but the dataset already "
                             "exists."
                          << THROW;
            }
            if (att.type[0] != MemcomTypeName<T>::name[0]) {
                DBError() << "Cannot write dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name[0] << "." << THROW;
            }
            if (att.ndim != 1) {
                DBError() << "Cannot write dataset " << key << ", it has " << att.ndim
                          << " instead of 1 dimensions." << THROW;
            }
            if (att.dim[0] != mcOff((s1.is_all() ? v.size() : s1.size()))) {
                DBError() << "Cannot write dataset " << key << ", its first dimension is "
                          << att.dim[0] << " instead of " << (s1.is_all() ? v.size() : s1.size())
                          << "." << THROW;
            }
        }
        if (s1.is_all()) {
            int status = mcDBputSet(
                  handler, key.c_str(), MemcomTypeName<T>::name, v.size(), const_cast<T*>(&v[0]));
            if (status != 0) {
                DBError() << "mcDBputSetpos returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl;
            s1.set_mcslice(&sl);
            int status = mcDBputSetslice(
                  handler, key.c_str(), MemcomTypeName<T>::name, 1, &sl, 0, 0,
                  const_cast<T*>(&v[0]));
            if (status != 0) {
                DBError() << "mcDBputSetslice returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
    }
 
    template <typename T>
    void set(
          const std::string& key, const b2linalg::Vector<T, b2linalg::Vincrement_constref>& v,
          const DBSlice& s1, const DBSlice& s2, bool new_set = false) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            if (!s1.is_all() || !s2.is_all()) {
                DBError() << "Cannot write dataset " << key
                          << ", an existing dataset is requested, but the dataset "
                             "does not exist."
                          << THROW;
            }
            mcOff dim = v.size();
            int status = mcDBcreateSet(handler, key.c_str(), MemcomTypeName<T>::name, 1, &dim);
            if (status != 0) {
                DBError() << "mcDBcreateSet returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            if (new_set) {
                DBError() << "Cannot write dataset " << key
                          << ", a new dataset is requested but the dataset already "
                             "exists."
                          << THROW;
            }
            if (att.type[0] != MemcomTypeName<T>::name[0]) {
                DBError() << "Cannot write dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name[0] << "." << THROW;
            }
            if (s2.is_null()) {
                if (att.ndim != 1) {
                    DBError() << "Cannot write dataset" << key << ", it has " << att.ndim
                              << " instead of 1 dimensions." << THROW;
                }
            } else {
                if (att.ndim != 2) {
                    DBError() << "Cannot write dataset " << key << ", it has " << att.ndim
                              << " instead of 2 dimensions." << THROW;
                }
                if (att.dim[1] > s2.end) {
                    DBError() << "Cannot write dataset " << key << ", its second dimension is "
                              << att.dim[0] << " but should be less than " << s2.end << "."
                              << THROW;
                }
            }
            if (att.dim[0] != mcOff(s1.is_all() ? v.size() : s1.size())) {
                DBError() << "Cannot write dataset " << key << ", its first dimension is "
                          << att.dim[0] << " instead of " << (s1.is_all() ? v.size() : s1.size())
                          << "." << THROW;
            }
        }
        if (s1.is_all() && s2.is_all()) {
            int status = mcDBputSet(
                  handler, key.c_str(), MemcomTypeName<T>::name, v.size(), const_cast<T*>(&v[0]));
            if (status != 0) {
                DBError() << "mcDBputSetpos returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl[2];
            s2.set_mcslice(sl);
            s1.set_mcslice(sl + 1);
            int status = mcDBputSetslice(
                  handler, key.c_str(), MemcomTypeName<T>::name, 2, sl, 0, 0,
                  const_cast<T*>(&v[0]));
            if (status != 0) {
                DBError() << "mcDBputSetslice returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
    }
 
    template <typename T>
    void set(
          const std::string& key, const b2linalg::Matrix<T, b2linalg::Mrectangle>& m,
          const DBSlice& s1 = DBSlice::all, const DBSlice& s2 = DBSlice::all,
          bool new_set = false) {
        set(key, b2linalg::Matrix<T, b2linalg::Mrectangle_constref>(m), s1, s2, new_set);
    }
 
    template <typename T>
    void set(
          const std::string& key, const b2linalg::Matrix<T, b2linalg::Mrectangle_constref>& m,
          const DBSlice& s1 = DBSlice::all, const DBSlice& s2 = DBSlice::all,
          bool new_set = false) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            if (!s1.is_all() || !s2.is_all()) {
                DBError() << "Cannot write dataset " << key
                          << ", an existing dataset is requested, but the dataset "
                             "does not exist."
                          << THROW;
            }
            mcOff dim[2] = {mcOff(m.size2()), mcOff(m.size1())};
            int status = mcDBcreateSet(handler, key.c_str(), MemcomTypeName<T>::name, 2, dim);
            if (status != 0) {
                DBError() << "mcDBcreateSet returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            if (new_set) {
                DBError() << "Cannot write dataset " << key
                          << ", a new dataset is requested but the dataset already "
                             "exists."
                          << THROW;
            }
            if (att.type[0] != MemcomTypeName<T>::name[0]) {
                DBError() << "Cannot write dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name[0] << "." << THROW;
            }
            if (att.ndim != 2) {
                DBError() << "Cannot write dataset " << key << ", it has " << att.ndim
                          << " instead of 2 dimensions." << THROW;
            }
            if (att.dim[0] != mcOff(s1.is_all() ? m.size2() : s2.size())) {
                DBError() << "Cannot write dataset " << key << ", its first dimension is "
                          << att.dim[0] << " instead of " << (s1.is_all() ? m.size2() : s2.size())
                          << "." << THROW;
            }
            if (att.dim[1] != mcOff(s2.is_all() ? m.size1() : s1.size())) {
                DBError() << "Cannot write dataset " << key << " its second dimension is "
                          << att.dim[1] << " instead of " << (s2.is_all() ? m.size1() : s1.size())
                          << "." << THROW;
            }
        }
        if (s1.is_all() && s2.is_all()) {
            int status = mcDBputSet(
                  handler, key.c_str(), MemcomTypeName<T>::name, m.size1() * m.size2(),
                  const_cast<T*>(&m(0, 0)));
            if (status != 0) {
                DBError() << "mcDBputSetpos returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            mcSlice sl[2];
            s2.set_mcslice(sl);
            s1.set_mcslice(sl + 1);
            int status = mcDBputSetslice(
                  handler, key.c_str(), MemcomTypeName<T>::name, 2, sl, 0, 0,
                  const_cast<T*>(&m(0, 0)));
            if (status != 0) {
                DBError() << "mcDBputSetslice returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        }
    }
 
    template <typename T>
    void set(
          const std::string& key, const b2linalg::Matrix<T, b2linalg::Mpacked>& m,
          bool new_set = false) {
        mcSetAttributes att;
        if (mcDBinqSetAtt(handler, key.c_str(), &att) == 0) {
            mcOff dim[1] = {mcOff(m.esize())};
            int status = mcDBcreateSet(handler, key.c_str(), MemcomTypeName<T>::name, 1, dim);
            if (status != 0) {
                DBError() << "mcDBcreateSet returns an unexpected error: "
                          << mcErrGetMsgAndReset(status) << "." << THROW;
            }
        } else {
            if (new_set) {
                DBError() << "Cannot write dataset " << key
                          << ", a new dataset is requested but the dataset already "
                             "exists."
                          << THROW;
            }
            if (att.type[0] != MemcomTypeName<T>::name[0]) {
                DBError() << "Cannot write dataset " << key << ", its type is " << att.type
                          << " instead of " << MemcomTypeName<T>::name[0] << "." << THROW;
            }
            if (att.ndim != 1) {
                DBError() << "Cannot write dataset " << key << ", it has " << att.ndim
                          << " instead of 1 dimensions." << THROW;
            }
            if (att.dim[0] != mcOff(m.esize())) {
                DBError() << "Cannot write dataset " << key << ", its first dimension is "
                          << att.dim[0] << " instead of " << m.esize() << "." << THROW;
            }
        }
        int status = mcDBputSet(
              handler, key.c_str(), MemcomTypeName<T>::name, m.esize(), const_cast<T*>(&m(0, 0)));
        if (status != 0) {
            DBError() << "mcDBputSetpos returns an unexpected error: "
                      << mcErrGetMsgAndReset(status) << "." << THROW;
        }
    }
 
    void get(const std::string& key, RTable& rtable, const DBSlice& s1 = DBSlice::all) const;
 
    void set(const std::string& key, const RTable& rtable, bool new_set = false);
 
    void set(const std::string& key, const RTable& rtable, size_t pos);
 
    bool get_desc(
          const std::string& key, RTable& rtable,
          bool raise_exception_on_missing_desc = true) const;
 
    void set_desc(const std::string& key, const RTable& rtable);
 
    void get(const std::string& key, ArrayRTable& rtable) const;
 
    void set(
          const std::string& key, const std::vector<RTable>& array_rtable, bool new_set = false,
          bool new_sparse_table = false);
 
    class ArrayTableCol {
    public:
        template <typename T>
        ArrayTableCol(const std::string& col_name, const b2linalg::Matrix<T>& mat)
            : name(col_name),
              ncol(mat.size2()),
              nrow(mat.size1()),
              step(1),
              type_name(MemcomTypeName<T>::name),
              value(&mat(0, 0)) {}
 
        template <typename T>
        ArrayTableCol(const std::string& col_name, size_t col_size, const std::vector<T>& v)
            : name(col_name),
              ncol(col_size),
              nrow(v.size() / col_size),
              step(1),
              type_name(MemcomTypeName<T>::name),
              value(&v[0]) {}
 
        template <typename T>
        ArrayTableCol(
              const std::string& col_name, size_t col_size, size_t row_size, const T* v,
              size_t step)
            : name(col_name),
              ncol(col_size),
              nrow(row_size),
              step(step),
              type_name(MemcomTypeName<T>::name),
              value(v) {}
 
        void init_mcATColIndBuf(int column, mcATColIndBuf& att) const {
            att.column = column + 1;
            att.buffer = const_cast<void*>(value);
            att.buffer_step = step;
        }
        std::string name;
        size_t ncol;
        size_t nrow;
        size_t step;
        const char* type_name;
        const void* value;
    };
 
    void set(const std::string& key, const std::vector<ArrayTableCol>& cols, bool new_set = false) {
        {
            std::ostringstream o;
            o << cols[0].name << ":" << cols[0].ncol << cols[0].type_name;
            for (size_t i = 1; i < cols.size(); ++i) {
                if (cols[i].nrow != cols[0].nrow) { Exception() << THROW; }
                o << "." << cols[i].name << ":" << cols[i].ncol << cols[i].type_name;
            }
            std::string colspec = o.str();
            if (has_key(key)) {
                if (new_set) {
                    DBError() << "Cannot write dataset " << key
                              << ", a new dataset is requested but the dataset "
                                 "already exists."
                              << THROW;
                } else {
                    int status = mcDBdeleteSet(handler, key.c_str());
                    if (status != 0) { DBError() << mcErrGetMsgAndReset(status) << THROW; }
                }
            }
            int status = mcATcreateTable(handler, key.c_str(), cols[0].nrow, colspec.c_str());
            if (status != 0) { DBError() << mcErrGetMsgAndReset(status) << THROW; }
        }
        {
            std::vector<mcATColIndBuf> col_att(cols.size());
            for (size_t i = 0; i < cols.size(); ++i) { cols[i].init_mcATColIndBuf(i, col_att[i]); }
            if (mcATputSlice(
                      handler, key.c_str(), 1, cols[0].nrow, 1, col_att.size(), &col_att[0])) {
                DBError() << THROW;
            }
        }
    }
 
protected:
    std::string name;
    std::string db_name;
    int handler;
    bool is_readonly;
    bool close_db_on_delete;
    logging::Logger logger;
    Time last_sync;
    double time_sync;
 
    static bool can_convert_data(const std::string& type_from, const std::string& type_to) {
        if (type_from == "F") {
            if (type_to == "Z") { return true; }
            return false;
        } else if (type_from == "Z") {
            if (type_to == "F") { return true; }
            return false;
        }
        return false;
    }
 
    static void convert_data(
          const size_t num_elements, const std::string& type_from, const std::string& type_to,
          const void* data_from, void* data_to) {
        if (type_from == "F") {
            const double* data_from_t = (const double*)data_from;
            if (type_to == "Z") {
                b2000::csda<double>* data_to_t = (b2000::csda<double>*)data_to;
                std::copy_n(data_from_t, num_elements, data_to_t);
            } else {
                assert(0);
            }
        } else if (type_from == "Z") {
            const b2000::csda<double>* data_from_t = (const b2000::csda<double>*)data_from;
            if (type_to == "F") {
                double* data_to_t = (double*)data_to;
                std::copy_n(data_from_t, num_elements, data_to_t);
            } else {
                assert(0);
            }
        } else {
            assert(0);
        }
    }
};
 
class DataBaseFieldSet : public DataBase {
public:
    DataBaseFieldSet() : DataBase(), fields_loaded(false), fields_saved(true) {}
 
    void add_field_entry(
          const std::string& title, const std::string& gname, const std::string& type,
          const std::string& csys, const bool indexed, const bool discrete, const bool additive);
 
    void close() override {
        save_fields();
        DataBase::close();
    }
 
    void sync() override {
        save_fields();
        DataBase::sync();
    }
 
    void temporary_close() override {
        save_fields();
        fields_loaded = false;
        fields_saved = true;
        fields_rtables.clear();
        fields_indices.clear();
        DataBase::temporary_close();
    }
 
private:
    void save_fields();
 
    typedef std::map<std::string, int> FieldsIndices;
 
    bool fields_loaded;
    bool fields_saved;
    ArrayRTable fields_rtables;
    FieldsIndices fields_indices;
};
 
void remove_database(const std::string& name, bool remove_directory);
 
}  // namespace b2000
 
#endif