geoffsee/code-tokenizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update test suite

Browse Source
This commit is contained in:
Geoff Seemueller
2024-12-01 11:42:50 -05:00
parent 337e882767
commit 826c0b2d1d
2 changed files with 1 additions and 1204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1203
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

2
test/core.test.ts
View File

@@ -1,7 +1,7 @@
// test/core.test.ts // test/core.test.ts
import { describe, it, expect, beforeEach, spyOn } from 'bun:test'; import { describe, it, expect, beforeEach, spyOn } from 'bun:test';
import { TokenCleaner, MarkdownGenerator } from '../src'; import { TokenCleaner, MarkdownGenerator } from '../src';
import micromatch from 'micromatch'; import * as micromatch from 'micromatch';
import llama3Tokenizer from 'llama3-tokenizer-js'; import llama3Tokenizer from 'llama3-tokenizer-js';
import path from 'path'; import path from 'path';
import fs from 'fs/promises'; import fs from 'fs/promises';