geoffsee/code-tokenizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

convert project to typescript

Browse Source
This commit is contained in:
Geoff Seemueller
2024-11-21 13:23:45 -05:00
parent 1226a742b5
commit 88dced2c4d
11 changed files with 218 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
build.ts Normal file
View File

@@ -0,0 +1,5 @@
await Bun.build({
entrypoints: ['./src/cli.ts'],
outdir: './dist',
target: 'node',
});

BIN
bun.lockb Executable file
View File

Binary file not shown.

8
package.json
View File

@@ -10,7 +10,7 @@
"dist" "dist"
], ],
"scripts": { "scripts": {
"build": "rm -rf dist && mkdir dist && cp src/*.js dist/", "build": "rm -rf dist && bun run build.ts",
"test": "echo \"No tests specified\" && exit 0", "test": "echo \"No tests specified\" && exit 0",
"prepublishOnly": "npm run build", "prepublishOnly": "npm run build",
"dev": "npx .", "dev": "npx .",
@@ -31,6 +31,10 @@
"@eslint/js": "^9.14.0", "@eslint/js": "^9.14.0",
"eslint": "^9.14.0", "eslint": "^9.14.0",
"globals": "^15.12.0", "globals": "^15.12.0",
"prettier": "^3.3.3" "prettier": "^3.3.3",
"bun": "latest",
"@types/bun": "latest",
"@types/node": "^22.9.1",
"@types/micromatch": "^4.0.9"
} }
} }

183
src/MarkdownGenerator.js → src/MarkdownGenerator.ts
View File

@@ -1,4 +1,4 @@
// MarkdownGenerator.js // MarkdownGenerator.ts
import path from 'path'; import path from 'path';
import { execSync } from 'child_process'; import { execSync } from 'child_process';
@@ -6,17 +6,18 @@ import { readFile, writeFile } from 'fs/promises';
import llama3Tokenizer from 'llama3-tokenizer-js'; import llama3Tokenizer from 'llama3-tokenizer-js';
import { TokenCleaner } from './TokenCleaner.js'; import { TokenCleaner } from './TokenCleaner.js';
import micromatch from 'micromatch'; import micromatch from 'micromatch';
import fileTypeExclusions from './fileTypeExclusions.js';
import fileExclusions from './fileExclusions.js';
/** interface MarkdownGeneratorOptions {
* @typedef {Object} MarkdownGeneratorOptions dir?: string;
* @property {string} [dir='.'] - The directory to process files from outputFilePath?: string;
* @property {string} [outputFilePath='./prompt.md'] - Path where the output markdown file will be saved fileTypeExclusions?: Set<string>;
* @property {Set<string>} [fileTypeExclusions] - Set of file extensions to exclude fileExclusions?: string[];
* @property {string[]} [fileExclusions] - Array of specific files or patterns to exclude customPatterns?: Record<string, any>;
* @property {Object} [customPatterns] - Custom patterns for token cleaning customSecretPatterns?: Record<string, any>;
* @property {Object} [customSecretPatterns] - Custom patterns for identifying and redacting secrets verbose?: boolean;
* @property {boolean} [verbose=true] - Whether to log detailed information during processing }
*/
/** /**
* @class MarkdownGenerator * @class MarkdownGenerator
@@ -24,148 +25,26 @@ import micromatch from 'micromatch';
* It can exclude specific file types and files, clean tokens, and include todo lists. * It can exclude specific file types and files, clean tokens, and include todo lists.
*/ */
export class MarkdownGenerator { export class MarkdownGenerator {
private dir: string;
private outputFilePath: string;
private fileTypeExclusions: Set<string>;
private fileExclusions: string[];
private tokenCleaner: TokenCleaner;
private verbose: boolean;
/** /**
* Creates an instance of MarkdownGenerator. * Creates an instance of MarkdownGenerator.
* @param {MarkdownGeneratorOptions} [options={}] - Configuration options for the generator * @param {MarkdownGeneratorOptions} [options={}] - Configuration options for the generator
*/ */
constructor(options = {}) { constructor(options: MarkdownGeneratorOptions = {}) {
this.dir = options.dir || '.'; this.dir = options.dir || '.';
this.outputFilePath = options.outputFilePath || './prompt.md'; this.outputFilePath = options.outputFilePath || './prompt.md';
this.fileTypeExclusions = new Set( this.fileTypeExclusions = new Set(
options.fileTypeExclusions || [ options.fileTypeExclusions || fileTypeExclusions,
// Images
'.jpg',
'.jpeg',
'.png',
'.gif',
'.bmp',
'.svg',
'.webp',
'.tiff',
'.ico',
// Fonts
'.ttf',
'.woff',
'.woff2',
'.eot',
'.otf',
// Lock files
'.lock',
'.lockb',
// Config files
'.yaml',
'.yml',
'.toml',
'.conf',
// Binary and compiled
'.exe',
'.dll',
'.so',
'.dylib',
'.bin',
'.dat',
'.pyc',
'.pyo',
'.class',
'.jar',
// Archives
'.zip',
'.tar',
'.gz',
'.rar',
'.7z',
// Media
'.mp3',
'.mp4',
'.avi',
'.mov',
'.wav',
// Database
'.db',
'.sqlite',
'.sqlite3'
]
); );
this.fileExclusions = options.fileExclusions || [ this.fileExclusions = options.fileExclusions || fileExclusions;
// Config patterns
'**/.*rc',
'**/.*rc.{js,json,yaml,yml}',
'**/*.config.{js,ts}',
'**/tsconfig.json',
'**/tsconfig*.json',
'**/jsconfig.json',
'**/jsconfig*.json',
'**/package-lock.json',
'**/.prettierignore',
// Environment and variables
'**/.env*',
'**/*.vars',
'**/secrets.*',
// Version control
'**/.git*',
'**/.hg*',
'**/.svn*',
'**/CVS',
'**/.github/',
// CI/CD
'**/.gitlab-ci.yml',
'**/azure-pipelines.yml',
'**/jenkins*',
// Dependency directories
'**/node_modules/',
'**/target/',
'**/__pycache__/',
'**/venv/',
'**/.venv/',
'**/env/',
'**/build/',
'**/dist/',
'**/out/',
'**/bin/',
'**/obj/',
// Documentation
'**/README*',
'**/CHANGELOG*',
'**/CONTRIBUTING*',
'**/LICENSE*',
'**/docs/',
'**/documentation/',
// IDE and editors
'**/.{idea,vscode,eclipse,settings,zed,cursor}/',
'**/.project',
'**/.classpath',
'**/.factorypath',
// Test and data
'**/test{s,}/',
'**/spec/',
'**/fixtures/',
'**/testdata/',
'**/__tests__/',
'**/*.{test,spec}.*',
'**/coverage/',
'**/jest.config.*',
// Logs and temporary files
'**/logs/',
'**/tmp/',
'**/temp/',
'**/*.log'
];
this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns); this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns);
this.verbose = options.verbose !== undefined ? options.verbose : true; this.verbose = options.verbose !== undefined ? options.verbose : true;
@@ -177,7 +56,7 @@ export class MarkdownGenerator {
* @returns {Promise<string[]>} Array of tracked file paths that aren't excluded * @returns {Promise<string[]>} Array of tracked file paths that aren't excluded
* @throws {Error} When unable to execute git command or access files * @throws {Error} When unable to execute git command or access files
*/ */
async getTrackedFiles() { async getTrackedFiles(): Promise<string[]> {
try { try {
const output = this.execCommand('git ls-files'); const output = this.execCommand('git ls-files');
const trackedFiles = output.split('\n').filter(file => file.trim().length > 0); const trackedFiles = output.split('\n').filter(file => file.trim().length > 0);
@@ -210,7 +89,7 @@ export class MarkdownGenerator {
* @returns {Promise<string>} Cleaned and redacted content of the file * @returns {Promise<string>} Cleaned and redacted content of the file
* @throws {Error} When unable to read or process the file * @throws {Error} When unable to read or process the file
*/ */
async readFileContent(filePath) { async readFileContent(filePath: string): Promise<string> {
try { try {
const content = await readFile(filePath, 'utf-8'); const content = await readFile(filePath, 'utf-8');
const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content); const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content);
@@ -233,7 +112,7 @@ export class MarkdownGenerator {
* @returns {Promise<string>} Generated markdown content containing all processed files * @returns {Promise<string>} Generated markdown content containing all processed files
* @throws {Error} When unable to generate markdown content * @throws {Error} When unable to generate markdown content
*/ */
async generateMarkdown() { async generateMarkdown(): Promise<string> {
const trackedFiles = await this.getTrackedFiles(); const trackedFiles = await this.getTrackedFiles();
if (this.verbose) { if (this.verbose) {
console.log(`Generating markdown for ${trackedFiles.length} files`); console.log(`Generating markdown for ${trackedFiles.length} files`);
@@ -258,18 +137,18 @@ export class MarkdownGenerator {
* @returns {Promise<string>} Content of the todo file * @returns {Promise<string>} Content of the todo file
* @throws {Error} When unable to read or create the todo file * @throws {Error} When unable to read or create the todo file
*/ */
async getTodo() { async getTodo(): Promise<string> {
const todoPath = path.join(this.dir, 'todo'); const todoPath = path.join(this.dir, 'todo');
try { try {
if (this.verbose) { if (this.verbose) {
console.log('Reading todo file'); console.log('Reading todo file');
} }
return await readFile(todoPath, 'utf-8'); return await readFile(todoPath, 'utf-8');
} catch (error) { } catch (error: any) {
if (error.code === 'ENOENT') { if (error.code === 'ENOENT') {
// File does not exist // File does not exist
if (this.verbose) { if (this.verbose) {
console.log("File not found, creating a new 'todo' file."); console.log('File not found, creating a new \'todo\' file.');
} }
await writeFile(todoPath, ''); // Create an empty 'todo' file await writeFile(todoPath, ''); // Create an empty 'todo' file
return await this.getTodo(); // Await the recursive call return await this.getTodo(); // Await the recursive call
@@ -290,7 +169,7 @@ export class MarkdownGenerator {
* @returns {Error} [result.error] - Error object if operation failed * @returns {Error} [result.error] - Error object if operation failed
* @throws {Error} When unable to create or write the markdown document * @throws {Error} When unable to create or write the markdown document
*/ */
async createMarkdownDocument() { async createMarkdownDocument(): Promise<{ success: boolean, tokenCount?: number, error?: Error }> {
try { try {
const codeMarkdown = await this.generateMarkdown(); const codeMarkdown = await this.generateMarkdown();
const todos = await this.getTodo(); const todos = await this.getTodo();
@@ -317,7 +196,7 @@ export class MarkdownGenerator {
* @throws {Error} When command execution fails * @throws {Error} When command execution fails
* @private * @private
*/ */
execCommand(command) { private execCommand(command: string): string {
try { try {
return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim(); return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim();
} catch (error) { } catch (error) {
@@ -327,4 +206,4 @@ export class MarkdownGenerator {
throw error; throw error;
} }
} }
} }

16
src/TokenCleaner.js → src/TokenCleaner.ts
View File

@@ -1,5 +1,11 @@
export class TokenCleaner { export class TokenCleaner {
constructor(customPatterns = [], customSecretPatterns = []) { patterns: { regex: RegExp; replacement: string }[];
secretPatterns: { regex: RegExp; replacement: string }[];
constructor(customPatterns: { regex: RegExp; replacement: string }[] = [], customSecretPatterns: {
regex: RegExp;
replacement: string
}[] = []) {
this.patterns = [ this.patterns = [
{ regex: /\/\/.*$/gm, replacement: '' }, { regex: /\/\/.*$/gm, replacement: '' },
{ regex: /\/\*[\s\S]*?\*\//gm, replacement: '' }, { regex: /\/\*[\s\S]*?\*\//gm, replacement: '' },
@@ -40,22 +46,22 @@ export class TokenCleaner {
]; ];
} }
clean(code) { clean(code: string): string {
return this.patterns.reduce( return this.patterns.reduce(
(cleanCode, pattern) => cleanCode.replace(pattern.regex, pattern.replacement), (cleanCode, pattern) => cleanCode.replace(pattern.regex, pattern.replacement),
code, code,
); );
} }
redactSecrets(code) { redactSecrets(code: string): string {
return this.secretPatterns.reduce( return this.secretPatterns.reduce(
(redactedCode, pattern) => redactedCode.replace(pattern.regex, pattern.replacement), (redactedCode, pattern) => redactedCode.replace(pattern.regex, pattern.replacement),
code, code,
); );
} }
cleanAndRedact(code) { cleanAndRedact(code: string): string {
const cleanedCode = this.clean(code); const cleanedCode = this.clean(code);
return this.redactSecrets(cleanedCode); return this.redactSecrets(cleanedCode);
} }
} }

8
src/cli.js → src/cli.ts Executable file → Normal file
View File

@@ -1,16 +1,16 @@
#!/usr/bin/env node #!/usr/bin/env node
console.log("RUNNING TOKENIZER") console.log('RUNNING TOKENIZER');
import { MarkdownGenerator } from './MarkdownGenerator.js'; import { MarkdownGenerator } from './MarkdownGenerator.js';
const generator = new MarkdownGenerator(); const generator = new MarkdownGenerator();
generator generator
.createMarkdownDocument() .createMarkdownDocument()
.then(result => { .then((result: { success: boolean }) => {
if (!result.success) { if (!result.success) {
process.exit(1); process.exit(1);
} }
}) })
.catch(error => { .catch((error: any) => {
console.error('Error:', error); console.error('Error:', error);
process.exit(1); process.exit(1);
}); });

71
src/fileExclusions.ts Normal file
View File

@@ -0,0 +1,71 @@
export default [
// Config patterns
'**/.*rc',
'**/.*rc.{js,json,yaml,yml}',
'**/*.config.{js,ts}',
'**/tsconfig.json',
'**/tsconfig*.json',
'**/jsconfig.json',
'**/jsconfig*.json',
'**/package-lock.json',
'**/.prettierignore',
// Environment and variables
'**/.env*',
'**/*.vars',
'**/secrets.*',
// Version control
'**/.git*',
'**/.hg*',
'**/.svn*',
'**/CVS',
'**/.github/',
// CI/CD
'**/.gitlab-ci.yml',
'**/azure-pipelines.yml',
'**/jenkins*',
// Dependency directories
'**/node_modules/',
'**/target/',
'**/__pycache__/',
'**/venv/',
'**/.venv/',
'**/env/',
'**/build/',
'**/dist/',
'**/out/',
'**/bin/',
'**/obj/',
// Documentation
'**/README*',
'**/CHANGELOG*',
'**/CONTRIBUTING*',
'**/LICENSE*',
'**/docs/',
'**/documentation/',
// IDE and editors
'**/.{idea,vscode,eclipse,settings,zed,cursor}/',
'**/.project',
'**/.classpath',
'**/.factorypath',
// Test and data
'**/test{s,}/',
'**/spec/',
'**/fixtures/',
'**/testdata/',
'**/__tests__/',
'**/*.{test,spec}.*',
'**/coverage/',
'**/jest.config.*',
// Logs and temporary files
'**/logs/',
'**/tmp/',
'**/temp/',
'**/*.log'
]

60
src/fileTypeExclusions.ts Normal file
View File

@@ -0,0 +1,60 @@
export default [
// Images
'.jpg',
'.jpeg',
'.png',
'.gif',
'.bmp',
'.svg',
'.webp',
'.tiff',
'.ico',
// Fonts
'.ttf',
'.woff',
'.woff2',
'.eot',
'.otf',
// Lock files
'.lock',
'.lockb',
// Config files
'.yaml',
'.yml',
'.toml',
'.conf',
// Binary and compiled
'.exe',
'.dll',
'.so',
'.dylib',
'.bin',
'.dat',
'.pyc',
'.pyo',
'.class',
'.jar',
// Archives
'.zip',
'.tar',
'.gz',
'.rar',
'.7z',
// Media
'.mp3',
'.mp4',
'.avi',
'.mov',
'.wav',
// Database
'.db',
'.sqlite',
'.sqlite3'
]

2
src/index.js
View File

@@ -1,2 +0,0 @@
export { TokenCleaner } from './TokenCleaner.js';
export { MarkdownGenerator } from './MarkdownGenerator.js';

2
src/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export { TokenCleaner } from './TokenCleaner';
export { MarkdownGenerator } from './MarkdownGenerator';

28
tsconfig.json Normal file
View File

@@ -0,0 +1,28 @@
{
"compilerOptions": {
// Enable latest features
"lib": ["ESNext", "DOM"],
"target": "ESNext",
"module": "ESNext",
"moduleDetection": "force",
"jsx": "react-jsx",
"allowJs": true,
// Bundler mode
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"verbatimModuleSyntax": true,
"noEmit": true,
t
// Best practices
"strict": true,
"skipLibCheck": true,
"noFallthroughCasesInSwitch": true,
// Some stricter flags (disabled by default)
"noUnusedLocals": false,
"noUnusedParameters": false,
"noPropertyAccessFromIndexSignature": false
}
}