geoffsee/code-tokenizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add eslint

Browse Source
This commit is contained in:
Geoff Seemueller
2024-11-07 11:37:56 -05:00
parent 2f809fb82e
commit e747f99984
9 changed files with 906 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
src/MarkdownGenerator.js
View File

@@ -6,109 +6,142 @@ import llama3Tokenizer from 'llama3-tokenizer-js';
import { TokenCleaner } from './TokenCleaner.js';
export class MarkdownGenerator {
constructor(options = {}) {
this.dir = options.dir || '.';
this.outputFilePath = options.outputFilePath || './prompt.md';
this.fileTypeExclusions = new Set(options.fileTypeExclusions || ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp', '.tiff', '.lockb', '.yaml', '.ico', '.ttf', '.css']);
this.fileExclusions = options.fileExclusions || ['prompt.js', '.gitignore', '.env', '.dev.vars'];
this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns);
this.verbose = options.verbose ?? true;
}
constructor(options = {}) {
this.dir = options.dir || '.';
this.outputFilePath = options.outputFilePath || './prompt.md';
this.fileTypeExclusions = new Set(
options.fileTypeExclusions || [
'.jpg',
'.jpeg',
'.png',
'.gif',
'.bmp',
'.svg',
'.webp',
'.tiff',
'.lockb',
'.yaml',
'.ico',
'.ttf',
'.css',
],
);
this.fileExclusions = options.fileExclusions || [
'prompt.js',
'.gitignore',
'.env',
'.dev.vars',
];
this.tokenCleaner = new TokenCleaner(options.customPatterns, options.customSecretPatterns);
this.verbose = options.verbose ?? true;
}
async getTrackedFiles() {
try {
const output = this.execCommand('git ls-files');
const trackedFiles = output.split('\n').filter(file => file.length > 0);
if (this.verbose) console.log(`Total tracked files: ${trackedFiles.length}`);
return trackedFiles.filter(file => {
const fileExt = path.extname(file).toLowerCase();
const isExcluded = this.fileExclusions.some(pattern => this.isFileExcluded(file, pattern));
return !this.fileTypeExclusions.has(fileExt) && !isExcluded;
});
} catch (error) {
if (this.verbose) console.error('Error fetching tracked files:', error);
return [];
}
async getTrackedFiles() {
try {
const output = this.execCommand('git ls-files');
const trackedFiles = output.split('\n').filter(file => file.length > 0);
if (this.verbose) {
console.log(`Total tracked files: ${trackedFiles.length}`);
}
return trackedFiles.filter(file => {
const fileExt = path.extname(file).toLowerCase();
const isExcluded = this.fileExclusions.some(pattern => this.isFileExcluded(file, pattern));
return !this.fileTypeExclusions.has(fileExt) && !isExcluded;
});
} catch (error) {
if (this.verbose) {
console.error('Error fetching tracked files:', error);
}
return [];
}
}
isFileExcluded(filePath, pattern) {
if (pattern.endsWith('/*')) {
const directory = pattern.slice(0, -2);
return filePath.startsWith(directory);
}
if (pattern.includes('/*')) {
const [directory, ext] = pattern.split('/*');
return filePath.startsWith(directory) && filePath.endsWith(ext);
}
return filePath === pattern;
isFileExcluded(filePath, pattern) {
if (pattern.endsWith('/*')) {
const directory = pattern.slice(0, -2);
return filePath.startsWith(directory);
}
async readFileContent(filePath) {
try {
const content = await fs.readFile(filePath, 'utf-8');
const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content);
if (this.verbose) {
const tokenCount = llama3Tokenizer.encode(cleanedAndRedactedContent).length;
console.log(`${filePath}: Tokens[${tokenCount}]`);
}
return cleanedAndRedactedContent;
} catch (error) {
if (this.verbose) console.error(`Error reading file ${filePath}:`, error);
return '';
}
if (pattern.includes('/*')) {
const [directory, ext] = pattern.split('/*');
return filePath.startsWith(directory) && filePath.endsWith(ext);
}
return filePath === pattern;
}
async generateMarkdown() {
const trackedFiles = await this.getTrackedFiles();
if (this.verbose) console.log(`Generating markdown for ${trackedFiles.length} files`);
let markdownContent = '# Project Files\n\n';
for (const file of trackedFiles) {
const content = await this.readFileContent(path.join(this.dir, file));
markdownContent += `## ${file}\n~~~\n${content.trim()}\n~~~\n`;
}
return markdownContent;
async readFileContent(filePath) {
try {
const content = await fs.readFile(filePath, 'utf-8');
const cleanedAndRedactedContent = this.tokenCleaner.cleanAndRedact(content);
if (this.verbose) {
const tokenCount = llama3Tokenizer.encode(cleanedAndRedactedContent).length;
console.log(`${filePath}: Tokens[${tokenCount}]`);
}
return cleanedAndRedactedContent;
} catch (error) {
if (this.verbose) {
console.error(`Error reading file ${filePath}:`, error);
}
return '';
}
}
async getTodo() {
try {
console.log("getting project todo")
return await readFile('./todo', 'utf-8');
} catch (error) {
if (error.code === 'ENOENT') { // File does not exist
console.log("File not found, creating a new 'todo' file.");
await writeFile('./todo', ''); // Create an empty 'todo' file
return this.getTodo(); // Call the function again
} else {
console.error(`Error reading todo file:`, error);
}
}
async generateMarkdown() {
const trackedFiles = await this.getTrackedFiles();
if (this.verbose) {
console.log(`Generating markdown for ${trackedFiles.length} files`);
}
let markdownContent = '# Project Files\n\n';
async createMarkdownDocument() {
try {
const codeMarkdown = await this.generateMarkdown();
const todos = await this.getTodo();
const markdown = codeMarkdown + `\n---\n${todos}\n`;
await fs.writeFile(this.outputFilePath, markdown);
if (this.verbose) {
console.log(`Markdown document created at ${this.outputFilePath}`);
const totalTokens = llama3Tokenizer.encode(markdown).length;
console.log({total_tokens: totalTokens});
}
return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length };
} catch (error) {
if (this.verbose) console.error('Error writing markdown document:', error);
return { success: false, error };
}
for (const file of trackedFiles) {
const content = await this.readFileContent(path.join(this.dir, file));
markdownContent += `## ${file}\n~~~\n${content.trim()}\n~~~\n`;
}
return markdownContent;
}
execCommand(command) {
try {
return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim();
} catch (error) {
if (this.verbose) console.error(`Error executing command: ${command}`, error);
throw error;
}
async getTodo() {
try {
console.log('getting project todo');
return await readFile('./todo', 'utf-8');
} catch (error) {
if (error.code === 'ENOENT') {
// File does not exist
console.log("File not found, creating a new 'todo' file.");
await writeFile('./todo', ''); // Create an empty 'todo' file
return this.getTodo(); // Call the function again
}
console.error('Error reading todo file:', error);
}
}
}
async createMarkdownDocument() {
try {
const codeMarkdown = await this.generateMarkdown();
const todos = await this.getTodo();
const markdown = codeMarkdown + `\n---\n${todos}\n`;
await fs.writeFile(this.outputFilePath, markdown);
if (this.verbose) {
console.log(`Markdown document created at ${this.outputFilePath}`);
const totalTokens = llama3Tokenizer.encode(markdown).length;
console.log({ total_tokens: totalTokens });
}
return { success: true, tokenCount: llama3Tokenizer.encode(markdown).length };
} catch (error) {
if (this.verbose) {
console.error('Error writing markdown document:', error);
}
return { success: false, error };
}
}
execCommand(command) {
try {
return execSync(command, { cwd: this.dir, encoding: 'utf-8' }).toString().trim();
} catch (error) {
if (this.verbose) {
console.error(`Error executing command: ${command}`, error);
}
throw error;
}
}
}

93
src/TokenCleaner.js
View File

@@ -1,40 +1,61 @@
export class TokenCleaner {
constructor(customPatterns = [], customSecretPatterns = []) {
this.patterns = [
{ regex: /\/\/.*$/gm, replacement: '' },
{ regex: /\/\*[\s\S]*?\*\//gm, replacement: '' },
{ regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' },
{ regex: /^\s*[\r\n]/gm, replacement: '' },
{ regex: / +$/gm, replacement: '' },
{ regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' },
{ regex: /^\s*\n+/gm, replacement: '\n' },
...customPatterns
];
constructor(customPatterns = [], customSecretPatterns = []) {
this.patterns = [
{ regex: /\/\/.*$/gm, replacement: '' },
{ regex: /\/\*[\s\S]*?\*\//gm, replacement: '' },
{ regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' },
{ regex: /^\s*[\r\n]/gm, replacement: '' },
{ regex: / +$/gm, replacement: '' },
{ regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' },
{ regex: /^\s*\n+/gm, replacement: '\n' },
...customPatterns,
];
// eslint-no-no-useless-escape
this.secretPatterns = [
{
regex:
/(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{
regex:
/(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^'"]+(?=['"])/gi,
replacement: '[REDACTED]',
},
{ regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' },
{
regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi,
replacement: '[REDACTED]',
},
{
regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g,
replacement: '[REDACTED_JWT]',
},
{ regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' },
{
regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g,
replacement: '[REDACTED_BASE64]',
},
...customSecretPatterns,
];
}
this.secretPatterns = [
{ regex: /(?<=(['"])(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)['"]:\s*['"])[^\'"]+(?=['"])/gi, replacement: '[REDACTED]' },
{ regex: /(?<=(?:api[_-]?key|api[_-]?secret|access[_-]?token|auth[_-]?token|client[_-]?secret|password|secret[_-]?key|private[_-]?key)\s*=\s*['"])[^\'"]+(?=['"])/gi, replacement: '[REDACTED]' },
{ regex: /(?<=bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' },
{ regex: /(?<=Authorization:\s*Bearer\s+)[a-zA-Z0-9\-._~+\/]+=*/gi, replacement: '[REDACTED]' },
{ regex: /(?<=eyJ)[A-Za-z0-9-_=]+\.eyJ[A-Za-z0-9-_=]+\.[A-Za-z0-9-_.+\/=]*/g, replacement: '[REDACTED_JWT]' },
{ regex: /([a-f0-9]{40}|[a-f0-9]{64})/gi, replacement: '[REDACTED_HASH]' },
{ regex: /(?<=[^A-Za-z0-9]|^)([A-Za-z0-9+\/]{40}|[A-Za-z0-9+\/]{64})(?=[^A-Za-z0-9]|$)/g, replacement: '[REDACTED_BASE64]' },
...customSecretPatterns
];
}
clean(code) {
return this.patterns.reduce(
(cleanCode, pattern) => cleanCode.replace(pattern.regex, pattern.replacement),
code,
);
}
clean(code) {
return this.patterns.reduce((cleanCode, pattern) =>
cleanCode.replace(pattern.regex, pattern.replacement), code);
}
redactSecrets(code) {
return this.secretPatterns.reduce(
(redactedCode, pattern) => redactedCode.replace(pattern.regex, pattern.replacement),
code,
);
}
redactSecrets(code) {
return this.secretPatterns.reduce((redactedCode, pattern) =>
redactedCode.replace(pattern.regex, pattern.replacement), code);
}
cleanAndRedact(code) {
const cleanedCode = this.clean(code);
return this.redactSecrets(cleanedCode);
}
}
cleanAndRedact(code) {
const cleanedCode = this.clean(code);
return this.redactSecrets(cleanedCode);
}
}

21
src/cli.js
View File

@@ -2,13 +2,14 @@
import { MarkdownGenerator } from './MarkdownGenerator.js';
const generator = new MarkdownGenerator();
generator.createMarkdownDocument()
.then(result => {
if (!result.success) {
process.exit(1);
}
})
.catch(error => {
console.error('Error:', error);
process.exit(1);
});
generator
.createMarkdownDocument()
.then(result => {
if (!result.success) {
process.exit(1);
}
})
.catch(error => {
console.error('Error:', error);
process.exit(1);
});

2
src/index.js
View File

@@ -1,2 +1,2 @@
export { TokenCleaner } from './TokenCleaner.js';
export { MarkdownGenerator } from './MarkdownGenerator.js';
export { MarkdownGenerator } from './MarkdownGenerator.js';