keras_model.cc

Go to the documentation of this file.

#include "keras_model.h"

#include "tbb/parallel_for.h"

#include <algorithm>
#include <fstream>
#include <math.h>
using namespace std;

std::vector<float> keras::read_1d_array(std::ifstream& fin, int cols)
{
  vector<float> arr;
  float tmp_float;
  char tmp_char;
  fin >> tmp_char; // for '['
  for (int n = 0; n < cols; ++n) {
    fin >> tmp_float;
    arr.push_back(tmp_float);
  }
  fin >> tmp_char; // for ']'
  return arr;
}

void keras::DataChunk2D::read_from_file(const std::string& fname)
{
  ifstream fin(fname.c_str());
  fin >> m_depth >> m_rows >> m_cols;

  for (int d = 0; d < m_depth; ++d) {
    vector<vector<float>> tmp_single_depth;
    for (int r = 0; r < m_rows; ++r) {
      vector<float> tmp_row = keras::read_1d_array(fin, m_cols);
      tmp_single_depth.push_back(tmp_row);
    }
    data.push_back(tmp_single_depth);
  }
  fin.close();
}

void keras::LayerConv2D::load_weights(std::ifstream& fin)
{
  char tmp_char = ' ';
  string tmp_str = "";
  float tmp_float;
  bool skip = false;
  fin >> m_kernels_cnt >> m_depth >> m_rows >> m_cols >> m_border_mode;
  if (m_border_mode == "[") {
    m_border_mode = "valid";
    skip = true;
  }

  cout << "LayerConv2D " << m_kernels_cnt << "x" << m_depth << "x" << m_rows << "x" << m_cols
       << " border_mode " << m_border_mode << endl;
  // reading kernel weights
  for (int k = 0; k < m_kernels_cnt; ++k) {
    vector<vector<vector<float>>> tmp_depths;
    for (int d = 0; d < m_depth; ++d) {
      vector<vector<float>> tmp_single_depth;
      for (int r = 0; r < m_rows; ++r) {
        if (!skip) { fin >> tmp_char; } // for '['
        else {
          skip = false;
        }
        vector<float> tmp_row;
        for (int c = 0; c < m_cols; ++c) {
          fin >> tmp_float;
          //cout << tmp_float << " ";
          tmp_row.push_back(tmp_float);
        }
        fin >> tmp_char; // for ']'
        tmp_single_depth.push_back(tmp_row);
      }
      tmp_depths.push_back(tmp_single_depth);
    }
    m_kernels.push_back(tmp_depths);
  }
  // reading kernel biases
  fin >> tmp_char; // for '['
  for (int k = 0; k < m_kernels_cnt; ++k) {
    fin >> tmp_float;
    m_bias.push_back(tmp_float);
  }
  fin >> tmp_char; // for ']'
}

void keras::LayerActivation::load_weights(std::ifstream& fin)
{
  fin >> m_activation_type;
  cout << "Activation type " << m_activation_type << endl;
}

void keras::LayerMaxPooling::load_weights(std::ifstream& fin)
{
  fin >> m_pool_x >> m_pool_y;
  cout << "MaxPooling " << m_pool_x << "x" << m_pool_y << endl;
}

void keras::LayerDense::load_weights(std::ifstream& fin)
{
  fin >> m_input_cnt >> m_neurons;
  float tmp_float;
  char tmp_char = ' ';
  for (int i = 0; i < m_input_cnt; ++i) {
    vector<float> tmp_n;
    fin >> tmp_char; // for '['
    for (int n = 0; n < m_neurons; ++n) {
      fin >> tmp_float;
      tmp_n.push_back(tmp_float);
    }
    fin >> tmp_char; // for ']'
    m_weights.push_back(tmp_n);
  }
  cout << "weights " << m_weights.size() << endl;
  fin >> tmp_char; // for '['
  for (int n = 0; n < m_neurons; ++n) {
    fin >> tmp_float;
    m_bias.push_back(tmp_float);
  }
  fin >> tmp_char; // for ']'
  cout << "bias " << m_bias.size() << endl;
}

keras::KerasModel::KerasModel(const string& input_fname)
{
  load_weights(input_fname);
}

keras::DataChunk* keras::LayerFlatten::compute_output(keras::DataChunk* dc)
{
  vector<vector<vector<float>>> im = dc->get_3d();

  size_t csize = im[0].size();
  size_t rsize = im[0][0].size();
  size_t size = im.size() * csize * rsize;
  keras::DataChunkFlat* out = new DataChunkFlat(size);
  float* y_ret = out->get_1d_rw().data();
  for (size_t i = 0, dst = 0; i < im.size(); ++i) {
    for (size_t j = 0; j < csize; ++j) {
      float* row = im[i][j].data();
      for (size_t k = 0; k < rsize; ++k) {
        y_ret[dst++] = row[k];
      }
    }
  }

  return out;
}

keras::DataChunk* keras::LayerMaxPooling::compute_output(keras::DataChunk* dc)
{
  vector<vector<vector<float>>> im = dc->get_3d();
  vector<vector<vector<float>>> y_ret;
  for (unsigned int i = 0; i < im.size(); ++i) {
    vector<vector<float>> tmp_y;
    for (unsigned int j = 0; j < (unsigned int)(im[0].size() / m_pool_x); ++j) {
      tmp_y.push_back(vector<float>((int)(im[0][0].size() / m_pool_y), 0.0));
    }
    y_ret.push_back(tmp_y);
  }
  for (unsigned int d = 0; d < y_ret.size(); ++d) {
    for (unsigned int x = 0; x < y_ret[0].size(); ++x) {
      unsigned int start_x = x * m_pool_x;
      unsigned int end_x = start_x + m_pool_x;
      for (unsigned int y = 0; y < y_ret[0][0].size(); ++y) {
        unsigned int start_y = y * m_pool_y;
        unsigned int end_y = start_y + m_pool_y;

        vector<float> values;
        for (unsigned int i = start_x; i < end_x; ++i) {
          for (unsigned int j = start_y; j < end_y; ++j) {
            values.push_back(im[d][i][j]);
          }
        }
        y_ret[d][x][y] = *max_element(values.begin(), values.end());
      }
    }
  }
  keras::DataChunk* out = new keras::DataChunk2D();
  out->set_data(y_ret);
  return out;
}

void keras::missing_activation_impl(const string& act)
{
  cout << "Activation " << act << " not defined!" << endl;
  cout << "Please add its implementation before use." << endl;
  exit(1);
}

keras::DataChunk* keras::LayerActivation::compute_output(keras::DataChunk* dc)
{

  if (dc->get_data_dim() == 3) {
    vector<vector<vector<float>>> y = dc->get_3d();
    if (m_activation_type == "relu") {
      for (unsigned int i = 0; i < y.size(); ++i) {
        for (unsigned int j = 0; j < y[0].size(); ++j) {
          for (unsigned int k = 0; k < y[0][0].size(); ++k) {
            if (y[i][j][k] < 0) y[i][j][k] = 0;
          }
        }
      }
    }
    else if (m_activation_type == "tanh") {
      for (unsigned int i = 0; i < y.size(); ++i) {
        for (unsigned int j = 0; j < y[0].size(); ++j) {
          for (unsigned int k = 0; k < y[0][0].size(); ++k) {
            y[i][j][k] = tanh(y[i][j][k]);
          }
        }
      }
    }
    else {
      keras::missing_activation_impl(m_activation_type);
    }

    keras::DataChunk* out = new keras::DataChunk2D();
    out->set_data(y);
    return out;
  }
  else if (dc->get_data_dim() == 1) { // flat data, use 1D
    vector<float> y = dc->get_1d();
    if (m_activation_type == "relu") {
      for (unsigned int k = 0; k < y.size(); ++k) {
        if (y[k] < 0) y[k] = 0;
      }
    }
    else if (m_activation_type == "softmax") {
      float sum = 0.0;
      for (unsigned int k = 0; k < y.size(); ++k) {
        y[k] = exp(y[k]);
        sum += y[k];
      }
      for (unsigned int k = 0; k < y.size(); ++k) {
        y[k] /= sum;
      }
    }
    else if (m_activation_type == "tanh") {
      for (unsigned int k = 0; k < y.size(); ++k) {
        y[k] = tanh(y[k]);
      }
    }
    else if (m_activation_type == "sigmoid") {
      for (unsigned int k = 0; k < y.size(); ++k) {
        y[k] = 1.0F / (1.0F + exp(-y[k]));
      }
    }
    else {
      keras::missing_activation_impl(m_activation_type);
    }

    keras::DataChunk* out = new DataChunkFlat();
    out->set_data(y);
    return out;
  }
  else {
    throw "data dim not supported";
  }

  return dc;
}

// with border mode = valid
void keras::conv_single_depth_valid(std::vector<std::vector<float>>& y, // accumulate here
                                    std::vector<std::vector<float>> const& im,
                                    std::vector<std::vector<float>> const& k)
{
  size_t k1_size = k.size(), k2_size = k[0].size();
  unsigned int st_x = (k1_size - 1) >> 1;
  unsigned int st_y = (k2_size - 1) >> 1;

  for (unsigned int i = st_x; i < im.size() - st_x; ++i) {
    for (unsigned int j = st_y; j < im[0].size() - st_y; ++j) {

      float sum = 0;
      for (unsigned int k1 = 0; k1 < k1_size; ++k1) {
        //const float * k_data = k[k1_size-k1-1].data();
        //const float * im_data = im[i-st_x+k1].data();
        for (unsigned int k2 = 0; k2 < k2_size; ++k2) {
          //sum += k_data[k2_size-k2-1] * im_data[j-st_y+k2];
          sum += k[k1_size - k1 - 1][k2_size - k2 - 1] * im[i - st_x + k1][j - st_y + k2];
        }
      }
      y[i - st_x][j - st_y] += sum;
    }
  }
}

// with border mode = same
void keras::conv_single_depth_same(std::vector<std::vector<float>>& y, // accumulate here
                                   std::vector<std::vector<float>> const& im,
                                   std::vector<std::vector<float>> const& k)
{
  size_t k1_size = k.size(), k2_size = k[0].size();
  unsigned int st_x = (k1_size - 1) >> 1;
  unsigned int st_y = (k2_size - 1) >> 1;

  size_t max_imc = im.size() - 1;
  size_t max_imr = im[0].size() - 1;

  for (unsigned int i = 0; i < im.size(); ++i) {
    for (unsigned int j = 0; j < im[0].size(); ++j) {

      float sum = 0;
      for (unsigned int k1 = 0; k1 < k1_size; ++k1) {
        //const float * k_data = k[k1_size-k1-1].data();
        //const float * im_data = im[i-st_x+k1].data();
        for (unsigned int k2 = 0; k2 < k2_size; ++k2) {
          if (i + k1 < st_x) continue;
          if (i + k1 > max_imc + st_x) continue;
          if (j + k2 < st_y) continue;
          if (j + k2 > max_imr + st_y) continue;

          //sum += k_data[k2_size-k2-1] * im_data[j-st_y+k2];
          sum += k[k1_size - k1 - 1][k2_size - k2 - 1] * im[i - st_x + k1][j - st_y + k2];
        }
      }
      y[i][j] += sum;
    }
  }
}

keras::DataChunk* keras::LayerConv2D::compute_output(keras::DataChunk* dc)
{
  unsigned int st_x = (m_kernels[0][0].size() - 1) >> 1;
  unsigned int st_y = (m_kernels[0][0][0].size() - 1) >> 1;
  auto const& im = dc->get_3d();

  size_t size_x = (m_border_mode == "valid") ? im[0].size() - 2 * st_x : im[0].size();
  size_t size_y = (m_border_mode == "valid") ? im[0][0].size() - 2 * st_y : im[0][0].size();

  //                                                    depth         rows    cols
  keras::DataChunk2D* out = new keras::DataChunk2D(m_kernels.size(), size_x, size_y, 0);
  auto& y_ret = out->get_3d_rw();

  //auto t1 = std::chrono::high_resolution_clock::now();

  // Parallelize the kernal calculation
  tbb::parallel_for(size_t(0), size_t(m_kernels.size()), [&](size_t j) {
    for (unsigned int m = 0; m < im.size(); ++m) { // loop over image depth
      if (m_border_mode == "valid")
        keras::conv_single_depth_valid(y_ret[j], im[m], m_kernels[j][m]);
      else
        keras::conv_single_depth_same(y_ret[j], im[m], m_kernels[j][m]);
    }

    for (unsigned int x = 0; x < y_ret[0].size(); ++x) {

      size_t size = y_ret[0][0].size();
      float bias = m_bias[j];
      size_t k = 0;

      for (unsigned int y = 0; y < size / 8; ++y) {
        y_ret[j][x][k] += bias;
        y_ret[j][x][k + 1] += bias;
        y_ret[j][x][k + 2] += bias;
        y_ret[j][x][k + 3] += bias;
        y_ret[j][x][k + 4] += bias;
        y_ret[j][x][k + 5] += bias;
        y_ret[j][x][k + 6] += bias;
        y_ret[j][x][k + 7] += bias;
        k += 8;
      }
      while (k < size) {
        y_ret[j][x][k] += bias;
        ++k;
      }
    }
  });

  //auto t2 = std::chrono::high_resolution_clock::now();
  /*
  cout << "Parallal : "
       << std::chrono::duration_cast<std::chrono::microseconds>(t2-t1).count()
       << " microseconds." << endl;
  */

  return out;
}

keras::DataChunk* keras::LayerDense::compute_output(keras::DataChunk* dc)
{
  //cout << "weights: input size " << m_weights.size() << endl;
  //cout << "weights: neurons size " << m_weights[0].size() << endl;
  //cout << "bias " << m_bias.size() << endl;
  size_t size = m_weights[0].size();
  size_t size8 = size >> 3;

  keras::DataChunkFlat* out = new DataChunkFlat(size, 0);
  float* y_ret = out->get_1d_rw().data();

  auto const& im = dc->get_1d();

  for (size_t j = 0; j < m_weights.size(); ++j) { // iter over input
    const float* w = m_weights[j].data();
    float p = im[j];
    size_t k = 0;
    for (size_t i = 0; i < size8; ++i) { // iter over neurons
      y_ret[k] += w[k] * p;              // vectorize if you can
      y_ret[k + 1] += w[k + 1] * p;
      y_ret[k + 2] += w[k + 2] * p;
      y_ret[k + 3] += w[k + 3] * p;
      y_ret[k + 4] += w[k + 4] * p;
      y_ret[k + 5] += w[k + 5] * p;
      y_ret[k + 6] += w[k + 6] * p;
      y_ret[k + 7] += w[k + 7] * p;
      k += 8;
    }
    while (k < size) {
      y_ret[k] += w[k] * p;
      ++k;
    }
  }
  for (size_t i = 0; i < size; ++i) { // add biases
    y_ret[i] += m_bias[i];
  }

  return out;
}

std::vector<float> keras::KerasModel::compute_output(keras::DataChunk* dc)
{
  //cout << endl << "KerasModel compute output" << endl;
  //cout << "Input data size:" << endl;
  //dc->show_name();

  keras::DataChunk* inp = dc;
  keras::DataChunk* out = 0;
  for (int l = 0; l < (int)m_layers.size(); ++l) {
    //cout << "Processing layer " << m_layers[l]->get_name() << endl;
    out = m_layers[l]->compute_output(inp);

    //cout << "Input" << endl;
    //inp->show_name();
    //cout << "Output" << endl;
    //out->show_name();

    if (inp != dc) delete inp;
    inp = 0L;
    inp = out;
  }

  //cout << "Output: ";
  //out->show_values();

  std::vector<float> flat_out = out->get_1d();
  delete out;

  return flat_out;
}

void keras::KerasModel::load_weights(const string& input_fname)
{
  cout << "Reading model from " << input_fname << endl;
  ifstream fin(input_fname.c_str());
  string layer_type = "";
  string tmp_str = "";
  int tmp_int = 0;

  fin >> tmp_str >> m_layers_cnt;
  cout << "Layers " << m_layers_cnt << endl;

  for (int layer = 0; layer < m_layers_cnt; ++layer) { // iterate over layers
    fin >> tmp_str >> tmp_int >> layer_type;
    cout << "Layer " << tmp_int << " " << layer_type << endl;

    Layer* l = 0L;
    if (layer_type == "Convolution2D") { l = new LayerConv2D(); }
    else if (layer_type == "Activation") {
      l = new LayerActivation();
    }
    else if (layer_type == "MaxPooling2D") {
      l = new LayerMaxPooling();
    }
    else if (layer_type == "Flatten") {
      l = new LayerFlatten();
    }
    else if (layer_type == "Dense") {
      l = new LayerDense();
    }
    else if (layer_type == "Dropout") {
      continue; // we dont need dropout layer in prediciton mode
    }
    if (l == 0L) {
      cout << "Layer is empty, maybe it is not defined? Cannot define network." << endl;
      return;
    }
    l->load_weights(fin);
    m_layers.push_back(l);
  }

  fin.close();
}

keras::KerasModel::~KerasModel()
{
  for (int i = 0; i < (int)m_layers.size(); ++i) {
    delete m_layers[i];
  }
}

int keras::KerasModel::get_output_length() const
{
  int i = m_layers.size() - 1;
  while ((i > 0) && (m_layers[i]->get_output_units() == 0))
    --i;
  return m_layers[i]->get_output_units();
}