splitIntoPotentialTokens.js 978 B

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. // \n = 10
  7. // ; = 59
  8. // { = 123
  9. // } = 125
  10. // <space> = 32
  11. // \r = 13
  12. // \t = 9
  13. /**
  14. * @param {string} str string
  15. * @returns {string[] | null} array of string separated by potential tokens
  16. */
  17. const splitIntoPotentialTokens = (str) => {
  18. const len = str.length;
  19. if (len === 0) return null;
  20. const results = [];
  21. let i = 0;
  22. while (i < len) {
  23. const start = i;
  24. block: {
  25. let cc = str.charCodeAt(i);
  26. while (cc !== 10 && cc !== 59 && cc !== 123 && cc !== 125) {
  27. if (++i >= len) break block;
  28. cc = str.charCodeAt(i);
  29. }
  30. while (
  31. cc === 59 ||
  32. cc === 32 ||
  33. cc === 123 ||
  34. cc === 125 ||
  35. cc === 13 ||
  36. cc === 9
  37. ) {
  38. if (++i >= len) break block;
  39. cc = str.charCodeAt(i);
  40. }
  41. if (cc === 10) {
  42. i++;
  43. }
  44. }
  45. results.push(str.slice(start, i));
  46. }
  47. return results;
  48. };
  49. module.exports = splitIntoPotentialTokens;