Subversion Repositories DIN Is Noise

/*
* tokenizer.cc
* DIN Is Noise is copyright (c) 2006-2025 Jagannathan Sampath
* DIN Is Noise is released under GNU Public License 2.0
* For more information, please visit https://dinisnoise.org/
*/


// string tokenizer
// based on initial code by Song Ho Ahn (song.ahn@gmail.com)

#include "tokenizer.h"
#include <cstdlib>
#include <iostream>
using namespace std;

tokenizer::tokenizer() {
  str ("");
}

tokenizer::tokenizer(const std::string& s) {
  str (s);
}

tokenizer::tokenizer (const std::vector<std::string>& t) {
  vec (t);
}

void tokenizer::str (const std::string& s) {
  buf = s;
  len = buf.length ();
  cur = 0;
  avail = 0;
}

void tokenizer::vec (const std::vector<std::string>& t) {
  tokens = t;
  ntok = tokens.size ();
  avail = 1;
  id = 0;
}

tokenizer& tokenizer::operator>> (std::string& s) {
  if (avail) {
    if (id < ntok)
      s = tokens [id++];
    else {
      s = "";
      cur = len;
    }
  } else {
    token = "";
    if (len < 1) {
      s = token;
    } else {
      skipws ();
      while (cur < len && (buf[cur] != delim)) {
        token += buf[cur];
        ++cur;
      }
      s = token;
    }
  }
  return *this;
}

tokenizer& tokenizer::operator>> (double& d) {
  string s; *this >> s;
  d = atof (s.c_str());
  return *this;
}

tokenizer& tokenizer::operator>> (float& f) {
  string s; *this >> s;
  f = (float) atof (s.c_str());
  return *this;
}

tokenizer& tokenizer::operator>> (int& i) {
  string s; *this >> s;
  i = atoi (s.c_str());
  return *this;
}

tokenizer& tokenizer::operator>> (char& c) {
  string s; *this >> s;
  if (s.length()) c = s[0];
  return *this;
}

tokenizer& tokenizer::operator>> (unsigned char& c) {
  string s; *this >> s;
  if (s.length()) c = s[0];
  return *this;
}

void tokenizer::skipws () {
  while (cur < len)
    if (buf[cur] == delim)  
      ++cur;
    else
      break;
}

string tokenizer::cur2end () {
  if (avail) {
    string result;
    for (int i = id, j = tokens.size(); i < j; ++i) result += (tokens[i] + spc);
    return result;
  } else {
    if (cur > -1)
      return buf.substr (cur, len - 1);
  }
  return "";
}