From 137976a17c05062ab64b13ac89d94fb218f61f1b Mon Sep 17 00:00:00 2001 From: Geoff Seemueller Date: Thu, 21 Nov 2024 13:36:57 -0500 Subject: [PATCH] generated test suite --- package.json | 2 +- src/TokenCleaner.ts | 14 +- test/core.test.ts | 457 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 465 insertions(+), 8 deletions(-) create mode 100644 test/core.test.ts diff --git a/package.json b/package.json index bc5227e..bfef484 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ ], "scripts": { "build": "rm -rf dist && bun run build.ts", - "test": "echo \"No tests specified\" && exit 0", + "test": "bun test", "prepublishOnly": "npm run build", "dev": "npx .", "deploy:dev": "pnpm build && pnpm publish .", diff --git a/src/TokenCleaner.ts b/src/TokenCleaner.ts index c4eb9eb..0bf0106 100644 --- a/src/TokenCleaner.ts +++ b/src/TokenCleaner.ts @@ -7,13 +7,13 @@ export class TokenCleaner { replacement: string }[] = []) { this.patterns = [ - { regex: /\/\/.*$/gm, replacement: '' }, - { regex: /\/\*[\s\S]*?\*\//gm, replacement: '' }, - { regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' }, - { regex: /^\s*[\r\n]/gm, replacement: '' }, - { regex: / +$/gm, replacement: '' }, - { regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, - { regex: /^\s*\n+/gm, replacement: '\n' }, + { regex: /\/\/.*$/gm, replacement: '' }, // Single-line comments + { regex: /\/\*[\s\S]*?\*\//g, replacement: '' }, // Multi-line comments + { regex: /console\.(log|error|warn|info)\(.*?\);?/g, replacement: '' }, // Console statements + { regex: /^\s*[\r\n]/gm, replacement: '' }, // Empty lines + { regex: / +$/gm, replacement: '' }, // Trailing spaces + { regex: /^\s*import\s+.*?;?\s*$/gm, replacement: '' }, // Import statements + { regex: /^\s*\n+/gm, replacement: '\n' }, // Multiple newlines ...customPatterns, ]; // eslint-no-no-useless-escape diff --git a/test/core.test.ts b/test/core.test.ts new file mode 100644 index 0000000..2c45ce7 --- /dev/null +++ b/test/core.test.ts @@ -0,0 +1,457 @@ +// test/core.test.ts +import { describe, it, expect, beforeEach, spyOn } from 'bun:test'; +import { TokenCleaner, MarkdownGenerator } from '../src'; +import micromatch from 'micromatch'; +import llama3Tokenizer from 'llama3-tokenizer-js'; +import path from 'path'; +import fs from 'fs/promises'; +import child_process from 'child_process'; + +describe('TokenCleaner', () => { + let tokenCleaner: TokenCleaner; + + beforeEach(() => { + tokenCleaner = new TokenCleaner(); + }); + + describe('clean', () => { + it('should remove single-line comments', () => { + const code = `const a = 1; // This is a comment +const b = 2;`; + const expected = `const a = 1; +const b = 2;`; + expect(tokenCleaner.clean(code)).toBe(expected); + }); + + it('should remove multi-line comments', () => { + const code = `/* This is a +multi-line comment */ +const a = 1;`; + const expected = `const a = 1;`; + expect(tokenCleaner.clean(code)).toBe(expected); + }); + + it('should remove console statements', () => { + const code = `console.log('Debugging'); +const a = 1;`; + const expected = ` +const a = 1;`; + expect(tokenCleaner.clean(code)).toBe(expected); + }); + + it('should remove import statements', () => { + const code = `import fs from 'fs'; +const a = 1;`; + const expected = ` +const a = 1;`; + expect(tokenCleaner.clean(code)).toBe(expected); + }); + + it('should trim whitespace and empty lines', () => { + const code = `const a = 1; + + +const b = 2; `; + const expected = `const a = 1; +const b = 2;`; + expect(tokenCleaner.clean(code)).toBe(expected); + }); + + it('should apply custom patterns', () => { + const customPatterns = [ + { regex: /DEBUG\s*=\s*true/g, replacement: 'DEBUG = false' }, + ]; + const customTokenCleaner = new TokenCleaner(customPatterns); + const code = `const DEBUG = true; +const a = 1;`; + const expected = `const DEBUG = false; +const a = 1;`; + expect(customTokenCleaner.clean(code)).toBe(expected); + }); + }); + + describe('redactSecrets', () => { + it('should redact API keys', () => { + const code = `const apiKey = '12345-ABCDE';`; + const expected = `const apiKey = '[REDACTED]';`; + expect(tokenCleaner.redactSecrets(code)).toBe(expected); + }); + + it('should redact bearer tokens', () => { + const code = `Authorization: Bearer abcdef123456`; + const expected = `Authorization: Bearer [REDACTED]`; + expect(tokenCleaner.redactSecrets(code)).toBe(expected); + }); + + it('should redact JWT tokens', () => { + const code = `const token = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.XmX8v1';`; + const expected = `const token = '[REDACTED_JWT]';`; + expect(tokenCleaner.redactSecrets(code)).toBe(expected); + }); + + it('should redact hashes', () => { + const code = `const hash = 'abcdef1234567890abcdef1234567890abcdef12';`; + const expected = `const hash = '[REDACTED_HASH]';`; + expect(tokenCleaner.redactSecrets(code)).toBe(expected); + }); + + it('should apply custom secret patterns', () => { + const customSecretPatterns = [ + { regex: /SECRET_KEY:\s*['"]([^'"]+)['"]/g, replacement: 'SECRET_KEY: [REDACTED]' }, + ]; + const customTokenCleaner = new TokenCleaner([], customSecretPatterns); + const code = `SECRET_KEY: 'mysecretkey123'`; + const expected = `SECRET_KEY: [REDACTED]`; + expect(customTokenCleaner.redactSecrets(code)).toBe(expected); + }); + }); + + describe('cleanAndRedact', () => { + it('should clean and redact code', () => { + const code = `// Comment +const apiKey = '12345-ABCDE'; +console.log('Debugging'); +import fs from 'fs'; + +/* Multi-line comment */ +const a = 1;`; + const expected = `const a = 1;`; + expect(tokenCleaner.cleanAndRedact(code)).toBe(expected); + }); + + it('should handle empty input', () => { + const code = ``; + expect(tokenCleaner.cleanAndRedact(code)).toBe(''); + }); + }); +}); + +describe('MarkdownGenerator', () => { + let markdownGenerator: MarkdownGenerator; + + beforeEach(() => { + markdownGenerator = new MarkdownGenerator({ verbose: false }); + }); + + describe('getTrackedFiles', () => { + it('should return filtered tracked files', async () => { + // Spy on execSync + const execSyncSpy = spyOn(child_process, 'execSync').mockImplementation(() => { + return `src/index.ts +src/MarkdownGenerator.ts +src/TokenCleaner.ts +README.md +node_modules/package.json +`; + }); + + // Spy on micromatch.isMatch + const isMatchSpy = spyOn(micromatch, 'isMatch').mockImplementation((file: string, pattern: string[]) => { + const excludedPatterns = [ + '**/.*rc', + '**/.*rc.{js,json,yaml,yml}', + '**tsconfig.json', + '**/tsconfig*.json', + '**/jsconfig.json', + '**/jsconfig*.json', + '**/package-lock.json', + '**/.prettierignore', + '**/.env*', + '**secrets.*', + '**/.git*', + '**/.hg*', + '**/.svn*', + '**/CVS', + '**/.github/', + '**/.gitlab-ci.yml', + '**/azure-pipelines.yml', + '**/jenkins*', + '**/node_modules/', + '**/target/', + '**/__pycache__/', + '**/venv/', + '**/.venv/', + '**/env/', + '**/build/', + '**/dist/', + '**/out/', + '**/bin/', + '**/obj/', + '**/README*', + '**/CHANGELOG*', + '**/CONTRIBUTING*', + '**/LICENSE*', + '**/docs/', + '**/documentation/', + '**/.{idea,vscode,eclipse,settings,zed,cursor}/', + '**/.project', + '**/.classpath', + '**/.factorypath', + '**/test{s,}/', + '**/spec/', + '**/fixtures/', + '**/testdata/', + '**/__tests__/', + '**coverage/', + '**/jest.config.*', + '**/logs/', + '**/tmp/', + '**/temp/', + '**/*.log', + ]; + + return excludedPatterns.some(pattern => micromatch.isMatch(file, pattern)); + }); + + const trackedFiles = await markdownGenerator.getTrackedFiles(); + expect(execSyncSpy).toHaveBeenCalledWith('git ls-files', { cwd: '.', encoding: 'utf-8' }); + expect(trackedFiles).toEqual([ + 'src/index.ts', + 'src/MarkdownGenerator.ts', + 'src/TokenCleaner.ts', + ]); + + // Restore the original implementations + execSyncSpy.mockRestore(); + isMatchSpy.mockRestore(); + }); + + it('should handle git command failure', async () => { + // Spy on execSync to throw an error + const execSyncSpy = spyOn(child_process, 'execSync').mockImplementation(() => { + throw new Error('Git command failed'); + }); + + const trackedFiles = await markdownGenerator.getTrackedFiles(); + expect(execSyncSpy).toHaveBeenCalled(); + expect(trackedFiles).toEqual([]); + + // Restore the original implementation + execSyncSpy.mockRestore(); + }); + }); + + describe('readFileContent', () => { + it('should read and clean file content', async () => { + const filePath = 'src/index.ts'; + const fileContent = `// This is a comment +const a = 1; // Inline comment +`; + const cleanedContent = `const a = 1;`; + + // Spy on fs.readFile + const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue(fileContent); + + // Spy on llama3Tokenizer.encode + const encodeSpy = spyOn(llama3Tokenizer, 'encode').mockReturnValue([1, 2, 3]); + + const content = await markdownGenerator.readFileContent(filePath); + expect(readFileSpy).toHaveBeenCalledWith(filePath, 'utf-8'); + expect(content).toBe(cleanedContent); + expect(encodeSpy).toHaveBeenCalledWith(cleanedContent); + + // Restore the original implementations + readFileSpy.mockRestore(); + encodeSpy.mockRestore(); + }); + + it('should handle readFile failure', async () => { + const filePath = 'src/missing.ts'; + + // Spy on fs.readFile to reject + const readFileSpy = spyOn(fs, 'readFile').mockRejectedValue(new Error('File not found')); + + const content = await markdownGenerator.readFileContent(filePath); + expect(readFileSpy).toHaveBeenCalledWith(filePath, 'utf-8'); + expect(content).toBe(''); + + // Restore the original implementation + readFileSpy.mockRestore(); + }); + }); + + describe('generateMarkdown', () => { + it('should generate markdown content from tracked files', async () => { + // Spy on getTrackedFiles + const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([ + 'src/index.ts', + 'src/MarkdownGenerator.ts', + ]); + + // Spy on readFileContent + const readFileContentSpy = spyOn(markdownGenerator, 'readFileContent').mockImplementation(async (filePath: string) => { + if (filePath === path.join('.', 'src/index.ts')) { + return `const a = 1;`; + } else if (filePath === path.join('.', 'src/MarkdownGenerator.ts')) { + return `class MarkdownGenerator {}`; + } + return ''; + }); + + const expectedMarkdown = `# Project Files + +## src/index.ts +~~~ +const a = 1; +~~~ + +## src/MarkdownGenerator.ts +~~~ +class MarkdownGenerator {} +~~~ + +`; + + const markdown = await markdownGenerator.generateMarkdown(); + expect(markdown).toBe(expectedMarkdown); + + // Restore the original implementations + getTrackedFilesSpy.mockRestore(); + readFileContentSpy.mockRestore(); + }); + + it('should handle no tracked files', async () => { + // Spy on getTrackedFiles + const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([]); + + const expectedMarkdown = `# Project Files + +`; + + const markdown = await markdownGenerator.generateMarkdown(); + expect(markdown).toBe(expectedMarkdown); + + // Restore the original implementation + getTrackedFilesSpy.mockRestore(); + }); + + it('should skip empty file contents', async () => { + // Spy on getTrackedFiles + const getTrackedFilesSpy = spyOn(markdownGenerator, 'getTrackedFiles').mockResolvedValue([ + 'src/index.ts', + 'src/empty.ts', + ]); + + // Spy on readFileContent + const readFileContentSpy = spyOn(markdownGenerator, 'readFileContent').mockImplementation(async (filePath: string) => { + if (filePath === path.join('.', 'src/index.ts')) { + return `const a = 1;`; + } else if (filePath === path.join('.', 'src/empty.ts')) { + return ` `; + } + return ''; + }); + + const expectedMarkdown = `# Project Files + +## src/index.ts +~~~ +const a = 1; +~~~ + +`; + + const markdown = await markdownGenerator.generateMarkdown(); + expect(markdown).toBe(expectedMarkdown); + + // Restore the original implementations + getTrackedFilesSpy.mockRestore(); + readFileContentSpy.mockRestore(); + }); + }); + + describe('getTodo', () => { + it('should read the todo file content', async () => { + const todoContent = `- [ ] Implement feature X +- [ ] Fix bug Y`; + + // Spy on fs.readFile + const readFileSpy = spyOn(fs, 'readFile').mockResolvedValue(todoContent); + + const todo = await markdownGenerator.getTodo(); + expect(readFileSpy).toHaveBeenCalledWith(path.join('.', 'todo'), 'utf-8'); + expect(todo).toBe(todoContent); + + // Restore the original implementation + readFileSpy.mockRestore(); + }); + + it('should create todo file if it does not exist', async () => { + const todoPath = path.join('.', 'todo'); + + // First call to readFile throws ENOENT, second call resolves to empty string + const readFileSpy = spyOn(fs, 'readFile') + .mockImplementationOnce(() => { + const error: any = new Error('File not found'); + error.code = 'ENOENT'; + return Promise.reject(error); + }) + .mockResolvedValueOnce(''); + + // Spy on fs.writeFile + const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); + + const todo = await markdownGenerator.getTodo(); + expect(readFileSpy).toHaveBeenCalledWith(todoPath, 'utf-8'); + expect(writeFileSpy).toHaveBeenCalledWith(todoPath, ''); + expect(readFileSpy).toHaveBeenCalledWith(todoPath, 'utf-8'); + expect(todo).toBe(''); + + // Restore the original implementations + readFileSpy.mockRestore(); + writeFileSpy.mockRestore(); + }); + + it('should throw error for non-ENOENT errors', async () => { + // Spy on fs.readFile to reject with a different error + const readFileSpy = spyOn(fs, 'readFile').mockRejectedValue({ code: 'EACCES' }); + + await expect(markdownGenerator.getTodo()).rejects.toEqual({ code: 'EACCES' }); + expect(readFileSpy).toHaveBeenCalledWith(path.join('.', 'todo'), 'utf-8'); + + // Restore the original implementation + readFileSpy.mockRestore(); + }); + }); + + describe('createMarkdownDocument', () => { + it('should create markdown document successfully', async () => { + // Spy on generateMarkdown and getTodo + const generateMarkdownSpy = spyOn(markdownGenerator, 'generateMarkdown').mockResolvedValue(`# Project Files`); + const getTodoSpy = spyOn(markdownGenerator, 'getTodo').mockResolvedValue(`---\n\n- [ ] Task 1`); + + // Spy on fs.writeFile + const writeFileSpy = spyOn(fs, 'writeFile').mockResolvedValue(undefined); + + // Spy on llama3Tokenizer.encode + const encodeSpy = spyOn(llama3Tokenizer, 'encode').mockReturnValue([1, 2, 3, 4]); + + const result = await markdownGenerator.createMarkdownDocument(); + expect(generateMarkdownSpy).toHaveBeenCalled(); + expect(getTodoSpy).toHaveBeenCalled(); + expect(writeFileSpy).toHaveBeenCalledWith( + './prompt.md', + `# Project Files\n\n---\n\n- [ ] Task 1\n` + ); + expect(result).toEqual({ success: true, tokenCount: 4 }); + + // Restore the original implementations + generateMarkdownSpy.mockRestore(); + getTodoSpy.mockRestore(); + writeFileSpy.mockRestore(); + encodeSpy.mockRestore(); + }); + + it('should handle errors during markdown creation', async () => { + // Spy on generateMarkdown to reject + const generateMarkdownSpy = spyOn(markdownGenerator, 'generateMarkdown').mockRejectedValue(new Error('Generation failed')); + + const result = await markdownGenerator.createMarkdownDocument(); + expect(result.success).toBe(false); + expect(result.error).toEqual(new Error('Generation failed')); + + // Restore the original implementation + generateMarkdownSpy.mockRestore(); + }); + }); +});