Subversion Repositories DIN Is Noise

/*
* tokenizer.cc
* DIN Is Noise is copyright (c) 2006-2021 Jagannathan Sampath
* DIN Is Noise is released under GNU Public License 2.0
* For more information, please visit https://dinisnoise.org/
*/


// string tokenizer
// based on initial code by Song Ho Ahn (song.ahn@gmail.com)

#include "tokenizer.h"
#include <cstdlib>

#include <iostream>
using namespace std;

tokenizer::tokenizer() {
  str ("");
  del (DEFAULT_DELIMITER);
}

tokenizer::tokenizer(const std::string& s, const std::string& d) {
  str (s);
  del (d);
}

tokenizer::tokenizer (const std::vector<std::string>& t) {
  vec (t);
}

void tokenizer::set (const std::string& s, const std::string& d) {
  str (s);
  del (d);
}

void tokenizer::str (const std::string& s) {
  buffer = s;
  blen = buffer.length ();
  init_cur ();
  tokens_available = 0;
}

void tokenizer::del (const std::string& d) {
  delimiter = d;
  dlen = delimiter.length ();
}

void tokenizer::vec (const std::vector<std::string>& t) {
  tokens = t;
  tokens_available = 1;
  tokid = 0;
}

void tokenizer::init_cur () {
  if (blen) cur = 0; else cur = -1;
}

tokenizer& tokenizer::operator>> (std::string& s) {
  if (tokens_available) {
    if (tokid < tokens.size()) s = tokens[tokid++]; else {
      s = "";
      cur = blen;
    }
  } else {
    token = "";
    if (blen < 1) {
      s = token;
    } else {
      skip_ws ();
      while ((cur < blen) && !isdelim (buffer[cur])) {
        token += buffer[cur];
        ++cur;
      }
      s = token;
    }
  }
  return *this;
}

tokenizer& tokenizer::operator>> (double& d) {
  string s; *this >> s;
  d = atof (s.c_str());
  return *this;
}

tokenizer& tokenizer::operator>> (float& f) {
  string s; *this >> s;
  f = (float) atof (s.c_str());
  return *this;
}

tokenizer& tokenizer::operator>> (int& i) {
  string s; *this >> s;
  i = atoi (s.c_str());
  return *this;
}

tokenizer& tokenizer::operator>> (char& c) {
  string s; *this >> s;
  if (s.length()) c = s[0];
  return *this;
}

tokenizer& tokenizer::operator>> (unsigned char& c) {
  string s; *this >> s;
  if (s.length()) c = s[0];
  return *this;
}

void tokenizer::skip_ws () {
  while (cur < blen) if (!isdelim (buffer[cur])) break; else ++cur;
}

bool tokenizer::isdelim (char c) {
  for (int i = 0; i < dlen; ++i) {
    if (delimiter[i] == c) return true;
  }
  return false;
}

string tokenizer::cur2end () {
  if (tokens_available) {
    string result;
    static const char spc = ' ';
    for (int i = tokid, j = tokens.size(); i < j; ++i) result = result + tokens[i] + spc;
    return result;
  } else {
    if (cur > -1) {
      return buffer.substr (cur, blen - 1);
    } else return "";
  }
}