deterministicGrouping.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. // Simulations show these probabilities for a single change
  7. // 93.1% that one group is invalidated
  8. // 4.8% that two groups are invalidated
  9. // 1.1% that 3 groups are invalidated
  10. // 0.1% that 4 or more groups are invalidated
  11. //
  12. // And these for removing/adding 10 lexically adjacent files
  13. // 64.5% that one group is invalidated
  14. // 24.8% that two groups are invalidated
  15. // 7.8% that 3 groups are invalidated
  16. // 2.7% that 4 or more groups are invalidated
  17. //
  18. // And these for removing/adding 3 random files
  19. // 0% that one group is invalidated
  20. // 3.7% that two groups are invalidated
  21. // 80.8% that 3 groups are invalidated
  22. // 12.3% that 4 groups are invalidated
  23. // 3.2% that 5 or more groups are invalidated
  24. /**
  25. * @param {string} a key
  26. * @param {string} b key
  27. * @returns {number} the similarity as number
  28. */
  29. const similarity = (a, b) => {
  30. const l = Math.min(a.length, b.length);
  31. let dist = 0;
  32. for (let i = 0; i < l; i++) {
  33. const ca = a.charCodeAt(i);
  34. const cb = b.charCodeAt(i);
  35. dist += Math.max(0, 10 - Math.abs(ca - cb));
  36. }
  37. return dist;
  38. };
  39. /**
  40. * @param {string} a key
  41. * @param {string} b key
  42. * @param {Set<string>} usedNames set of already used names
  43. * @returns {string} the common part and a single char for the difference
  44. */
  45. const getName = (a, b, usedNames) => {
  46. const l = Math.min(a.length, b.length);
  47. let i = 0;
  48. while (i < l) {
  49. if (a.charCodeAt(i) !== b.charCodeAt(i)) {
  50. i++;
  51. break;
  52. }
  53. i++;
  54. }
  55. while (i < l) {
  56. const name = a.slice(0, i);
  57. const lowerName = name.toLowerCase();
  58. if (!usedNames.has(lowerName)) {
  59. usedNames.add(lowerName);
  60. return name;
  61. }
  62. i++;
  63. }
  64. // names always contain a hash, so this is always unique
  65. // we don't need to check usedNames nor add it
  66. return a;
  67. };
  68. /**
  69. * @param {Record<string, number>} total total size
  70. * @param {Record<string, number>} size single size
  71. * @returns {void}
  72. */
  73. const addSizeTo = (total, size) => {
  74. for (const key of Object.keys(size)) {
  75. total[key] = (total[key] || 0) + size[key];
  76. }
  77. };
  78. /**
  79. * @param {Record<string, number>} total total size
  80. * @param {Record<string, number>} size single size
  81. * @returns {void}
  82. */
  83. const subtractSizeFrom = (total, size) => {
  84. for (const key of Object.keys(size)) {
  85. total[key] -= size[key];
  86. }
  87. };
  88. /**
  89. * @template T
  90. * @param {Iterable<Node<T>>} nodes some nodes
  91. * @returns {Record<string, number>} total size
  92. */
  93. const sumSize = nodes => {
  94. const sum = Object.create(null);
  95. for (const node of nodes) {
  96. addSizeTo(sum, node.size);
  97. }
  98. return sum;
  99. };
  100. /**
  101. * @param {Record<string, number>} size size
  102. * @param {Record<string, number>} maxSize minimum size
  103. * @returns {boolean} true, when size is too big
  104. */
  105. const isTooBig = (size, maxSize) => {
  106. for (const key of Object.keys(size)) {
  107. const s = size[key];
  108. if (s === 0) continue;
  109. const maxSizeValue = maxSize[key];
  110. if (typeof maxSizeValue === "number" && s > maxSizeValue) return true;
  111. }
  112. return false;
  113. };
  114. /**
  115. * @param {Record<string, number>} size size
  116. * @param {Record<string, number>} minSize minimum size
  117. * @returns {boolean} true, when size is too small
  118. */
  119. const isTooSmall = (size, minSize) => {
  120. for (const key of Object.keys(size)) {
  121. const s = size[key];
  122. if (s === 0) continue;
  123. const minSizeValue = minSize[key];
  124. if (typeof minSizeValue === "number" && s < minSizeValue) return true;
  125. }
  126. return false;
  127. };
  128. /**
  129. * @param {Record<string, number>} size size
  130. * @param {Record<string, number>} minSize minimum size
  131. * @returns {Set<string>} set of types that are too small
  132. */
  133. const getTooSmallTypes = (size, minSize) => {
  134. const types = new Set();
  135. for (const key of Object.keys(size)) {
  136. const s = size[key];
  137. if (s === 0) continue;
  138. const minSizeValue = minSize[key];
  139. if (typeof minSizeValue === "number" && s < minSizeValue) types.add(key);
  140. }
  141. return types;
  142. };
  143. /**
  144. * @template {object} T
  145. * @param {T} size size
  146. * @param {Set<string>} types types
  147. * @returns {number} number of matching size types
  148. */
  149. const getNumberOfMatchingSizeTypes = (size, types) => {
  150. let i = 0;
  151. for (const key of Object.keys(size)) {
  152. if (size[/** @type {keyof T} */ (key)] !== 0 && types.has(key)) i++;
  153. }
  154. return i;
  155. };
  156. /**
  157. * @param {Record<string, number>} size size
  158. * @param {Set<string>} types types
  159. * @returns {number} selective size sum
  160. */
  161. const selectiveSizeSum = (size, types) => {
  162. let sum = 0;
  163. for (const key of Object.keys(size)) {
  164. if (size[key] !== 0 && types.has(key)) sum += size[key];
  165. }
  166. return sum;
  167. };
  168. /**
  169. * @template T
  170. */
  171. class Node {
  172. /**
  173. * @param {T} item item
  174. * @param {string} key key
  175. * @param {Record<string, number>} size size
  176. */
  177. constructor(item, key, size) {
  178. this.item = item;
  179. this.key = key;
  180. this.size = size;
  181. }
  182. }
  183. /**
  184. * @template T
  185. */
  186. class Group {
  187. /**
  188. * @param {Node<T>[]} nodes nodes
  189. * @param {number[] | null} similarities similarities between the nodes (length = nodes.length - 1)
  190. * @param {Record<string, number>=} size size of the group
  191. */
  192. constructor(nodes, similarities, size) {
  193. this.nodes = nodes;
  194. this.similarities = similarities;
  195. this.size = size || sumSize(nodes);
  196. /** @type {string | undefined} */
  197. this.key = undefined;
  198. }
  199. /**
  200. * @param {(node: Node<T>) => boolean} filter filter function
  201. * @returns {Node<T>[] | undefined} removed nodes
  202. */
  203. popNodes(filter) {
  204. const newNodes = [];
  205. const newSimilarities = [];
  206. const resultNodes = [];
  207. let lastNode;
  208. for (let i = 0; i < this.nodes.length; i++) {
  209. const node = this.nodes[i];
  210. if (filter(node)) {
  211. resultNodes.push(node);
  212. } else {
  213. if (newNodes.length > 0) {
  214. newSimilarities.push(
  215. lastNode === this.nodes[i - 1]
  216. ? /** @type {number[]} */ (this.similarities)[i - 1]
  217. : similarity(/** @type {Node<T>} */ (lastNode).key, node.key)
  218. );
  219. }
  220. newNodes.push(node);
  221. lastNode = node;
  222. }
  223. }
  224. if (resultNodes.length === this.nodes.length) return;
  225. this.nodes = newNodes;
  226. this.similarities = newSimilarities;
  227. this.size = sumSize(newNodes);
  228. return resultNodes;
  229. }
  230. }
  231. /**
  232. * @template T
  233. * @param {Iterable<Node<T>>} nodes nodes
  234. * @returns {number[]} similarities
  235. */
  236. const getSimilarities = nodes => {
  237. // calculate similarities between lexically adjacent nodes
  238. /** @type {number[]} */
  239. const similarities = [];
  240. let last;
  241. for (const node of nodes) {
  242. if (last !== undefined) {
  243. similarities.push(similarity(last.key, node.key));
  244. }
  245. last = node;
  246. }
  247. return similarities;
  248. };
  249. /**
  250. * @template T
  251. * @typedef {object} GroupedItems<T>
  252. * @property {string} key
  253. * @property {T[]} items
  254. * @property {Record<string, number>} size
  255. */
  256. /**
  257. * @template T
  258. * @typedef {object} Options
  259. * @property {Record<string, number>} maxSize maximum size of a group
  260. * @property {Record<string, number>} minSize minimum size of a group (preferred over maximum size)
  261. * @property {Iterable<T>} items a list of items
  262. * @property {(item: T) => Record<string, number>} getSize function to get size of an item
  263. * @property {(item: T) => string} getKey function to get the key of an item
  264. */
  265. /**
  266. * @template T
  267. * @param {Options<T>} options options object
  268. * @returns {GroupedItems<T>[]} grouped items
  269. */
  270. module.exports = ({ maxSize, minSize, items, getSize, getKey }) => {
  271. /** @type {Group<T>[]} */
  272. const result = [];
  273. const nodes = Array.from(
  274. items,
  275. item => new Node(item, getKey(item), getSize(item))
  276. );
  277. /** @type {Node<T>[]} */
  278. const initialNodes = [];
  279. // lexically ordering of keys
  280. nodes.sort((a, b) => {
  281. if (a.key < b.key) return -1;
  282. if (a.key > b.key) return 1;
  283. return 0;
  284. });
  285. // return nodes bigger than maxSize directly as group
  286. // But make sure that minSize is not violated
  287. for (const node of nodes) {
  288. if (isTooBig(node.size, maxSize) && !isTooSmall(node.size, minSize)) {
  289. result.push(new Group([node], []));
  290. } else {
  291. initialNodes.push(node);
  292. }
  293. }
  294. if (initialNodes.length > 0) {
  295. const initialGroup = new Group(initialNodes, getSimilarities(initialNodes));
  296. /**
  297. * @param {Group<T>} group group
  298. * @param {Record<string, number>} consideredSize size of the group to consider
  299. * @returns {boolean} true, if the group was modified
  300. */
  301. const removeProblematicNodes = (group, consideredSize = group.size) => {
  302. const problemTypes = getTooSmallTypes(consideredSize, minSize);
  303. if (problemTypes.size > 0) {
  304. // We hit an edge case where the working set is already smaller than minSize
  305. // We merge problematic nodes with the smallest result node to keep minSize intact
  306. const problemNodes = group.popNodes(
  307. n => getNumberOfMatchingSizeTypes(n.size, problemTypes) > 0
  308. );
  309. if (problemNodes === undefined) return false;
  310. // Only merge it with result nodes that have the problematic size type
  311. const possibleResultGroups = result.filter(
  312. n => getNumberOfMatchingSizeTypes(n.size, problemTypes) > 0
  313. );
  314. if (possibleResultGroups.length > 0) {
  315. const bestGroup = possibleResultGroups.reduce((min, group) => {
  316. const minMatches = getNumberOfMatchingSizeTypes(min, problemTypes);
  317. const groupMatches = getNumberOfMatchingSizeTypes(
  318. group,
  319. problemTypes
  320. );
  321. if (minMatches !== groupMatches) {
  322. return minMatches < groupMatches ? group : min;
  323. }
  324. if (
  325. selectiveSizeSum(min.size, problemTypes) >
  326. selectiveSizeSum(group.size, problemTypes)
  327. ) {
  328. return group;
  329. }
  330. return min;
  331. });
  332. for (const node of problemNodes) bestGroup.nodes.push(node);
  333. bestGroup.nodes.sort((a, b) => {
  334. if (a.key < b.key) return -1;
  335. if (a.key > b.key) return 1;
  336. return 0;
  337. });
  338. } else {
  339. // There are no other nodes with the same size types
  340. // We create a new group and have to accept that it's smaller than minSize
  341. result.push(new Group(problemNodes, null));
  342. }
  343. return true;
  344. }
  345. return false;
  346. };
  347. if (initialGroup.nodes.length > 0) {
  348. const queue = [initialGroup];
  349. while (queue.length) {
  350. const group = /** @type {Group<T>} */ (queue.pop());
  351. // only groups bigger than maxSize need to be splitted
  352. if (!isTooBig(group.size, maxSize)) {
  353. result.push(group);
  354. continue;
  355. }
  356. // If the group is already too small
  357. // we try to work only with the unproblematic nodes
  358. if (removeProblematicNodes(group)) {
  359. // This changed something, so we try this group again
  360. queue.push(group);
  361. continue;
  362. }
  363. // find unsplittable area from left and right
  364. // going minSize from left and right
  365. // at least one node need to be included otherwise we get stuck
  366. let left = 1;
  367. const leftSize = Object.create(null);
  368. addSizeTo(leftSize, group.nodes[0].size);
  369. while (left < group.nodes.length && isTooSmall(leftSize, minSize)) {
  370. addSizeTo(leftSize, group.nodes[left].size);
  371. left++;
  372. }
  373. let right = group.nodes.length - 2;
  374. const rightSize = Object.create(null);
  375. addSizeTo(rightSize, group.nodes[group.nodes.length - 1].size);
  376. while (right >= 0 && isTooSmall(rightSize, minSize)) {
  377. addSizeTo(rightSize, group.nodes[right].size);
  378. right--;
  379. }
  380. // left v v right
  381. // [ O O O ] O O O [ O O O ]
  382. // ^^^^^^^^^ leftSize
  383. // rightSize ^^^^^^^^^
  384. // leftSize > minSize
  385. // rightSize > minSize
  386. // Perfect split: [ O O O ] [ O O O ]
  387. // right === left - 1
  388. if (left - 1 > right) {
  389. // We try to remove some problematic nodes to "fix" that
  390. let prevSize;
  391. if (right < group.nodes.length - left) {
  392. subtractSizeFrom(rightSize, group.nodes[right + 1].size);
  393. prevSize = rightSize;
  394. } else {
  395. subtractSizeFrom(leftSize, group.nodes[left - 1].size);
  396. prevSize = leftSize;
  397. }
  398. if (removeProblematicNodes(group, prevSize)) {
  399. // This changed something, so we try this group again
  400. queue.push(group);
  401. continue;
  402. }
  403. // can't split group while holding minSize
  404. // because minSize is preferred of maxSize we return
  405. // the problematic nodes as result here even while it's too big
  406. // To avoid this make sure maxSize > minSize * 3
  407. result.push(group);
  408. continue;
  409. }
  410. if (left <= right) {
  411. // when there is a area between left and right
  412. // we look for best split point
  413. // we split at the minimum similarity
  414. // here key space is separated the most
  415. // But we also need to make sure to not create too small groups
  416. let best = -1;
  417. let bestSimilarity = Infinity;
  418. let pos = left;
  419. const rightSize = sumSize(group.nodes.slice(pos));
  420. // pos v v right
  421. // [ O O O ] O O O [ O O O ]
  422. // ^^^^^^^^^ leftSize
  423. // rightSize ^^^^^^^^^^^^^^^
  424. while (pos <= right + 1) {
  425. const similarity = /** @type {number[]} */ (group.similarities)[
  426. pos - 1
  427. ];
  428. if (
  429. similarity < bestSimilarity &&
  430. !isTooSmall(leftSize, minSize) &&
  431. !isTooSmall(rightSize, minSize)
  432. ) {
  433. best = pos;
  434. bestSimilarity = similarity;
  435. }
  436. addSizeTo(leftSize, group.nodes[pos].size);
  437. subtractSizeFrom(rightSize, group.nodes[pos].size);
  438. pos++;
  439. }
  440. if (best < 0) {
  441. // This can't happen
  442. // but if that assumption is wrong
  443. // fallback to a big group
  444. result.push(group);
  445. continue;
  446. }
  447. left = best;
  448. right = best - 1;
  449. }
  450. // create two new groups for left and right area
  451. // and queue them up
  452. const rightNodes = [group.nodes[right + 1]];
  453. /** @type {number[]} */
  454. const rightSimilarities = [];
  455. for (let i = right + 2; i < group.nodes.length; i++) {
  456. rightSimilarities.push(
  457. /** @type {number[]} */ (group.similarities)[i - 1]
  458. );
  459. rightNodes.push(group.nodes[i]);
  460. }
  461. queue.push(new Group(rightNodes, rightSimilarities));
  462. const leftNodes = [group.nodes[0]];
  463. /** @type {number[]} */
  464. const leftSimilarities = [];
  465. for (let i = 1; i < left; i++) {
  466. leftSimilarities.push(
  467. /** @type {number[]} */ (group.similarities)[i - 1]
  468. );
  469. leftNodes.push(group.nodes[i]);
  470. }
  471. queue.push(new Group(leftNodes, leftSimilarities));
  472. }
  473. }
  474. }
  475. // lexically ordering
  476. result.sort((a, b) => {
  477. if (a.nodes[0].key < b.nodes[0].key) return -1;
  478. if (a.nodes[0].key > b.nodes[0].key) return 1;
  479. return 0;
  480. });
  481. // give every group a name
  482. const usedNames = new Set();
  483. for (let i = 0; i < result.length; i++) {
  484. const group = result[i];
  485. if (group.nodes.length === 1) {
  486. group.key = group.nodes[0].key;
  487. } else {
  488. const first = group.nodes[0];
  489. const last = group.nodes[group.nodes.length - 1];
  490. const name = getName(first.key, last.key, usedNames);
  491. group.key = name;
  492. }
  493. }
  494. // return the results
  495. return result.map(
  496. group =>
  497. /** @type {GroupedItems<T>} */
  498. ({
  499. key: group.key,
  500. items: group.nodes.map(node => node.item),
  501. size: group.size
  502. })
  503. );
  504. };