diff --git a/.code-tokenizer-md-ignore b/.toak-ignore similarity index 84% rename from .code-tokenizer-md-ignore rename to .toak-ignore index 7209a05..90c5869 100644 --- a/.code-tokenizer-md-ignore +++ b/.toak-ignore @@ -1,4 +1,4 @@ # This is just for testing to make sure the glob patterns work # Check is valid when test-for-ignore.css is not included in the program output **/*.css -**/.code-tokenizer-md-ignore \ No newline at end of file +**/.toak-ignore \ No newline at end of file diff --git a/README.md b/README.md index 25ec72a..1dcb620 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,22 @@ -# code-tokenizer-md -for me but you can use it too +# toak -![code-tokenizer-md](./code-tokenizer-md.jpg) - -> Push the limits of possible - -## Quick Start -```bash +```shell $ cd your-git-repo -$ npx code-tokenizer-md +$ npx toak ``` +![toak](./toak.jpg) + + ## Overview -`code-tokenizer-md` is a tool that processes git repository files, cleans code, redacts sensitive information, and generates markdown documentation with token counts using the Llama 3 tokenizer. +`toak` is a tool that processes git repository files, cleans code, redacts sensitive information, and generates markdown documentation with token counts using the Llama 3 tokenizer. +--- ## Philosophy > Human-first technologies for a better tomorrow. +--- ```mermaid graph TD @@ -41,7 +40,7 @@ graph TD - Removes comments, imports, and unnecessary whitespace - Redacts sensitive information (API keys, tokens, JWT, hashes) - Counts tokens using llama3-tokenizer-js -- Supports nested .code-tokenizer-md-ignore files +- Supports nested .toak-ignore files ### Token Cleaning - Removes single-line and multi-line comments @@ -65,20 +64,20 @@ graph TD ## Installation ```bash -npm install code-tokenizer-md +npm install toak ``` ## Usage ### CLI ```bash -npx code-tokenizer-md +npx toak ``` ### Programmatic Usage ```typescript -import { MarkdownGenerator } from 'code-tokenizer-md'; +import { MarkdownGenerator } from 'toak'; const generator = new MarkdownGenerator({ dir: './project', @@ -107,9 +106,9 @@ interface MarkdownGeneratorOptions { ### Ignore File Configuration -Create a `.code-tokenizer-md-ignore` file in any directory to specify exclusions. The tool supports nested ignore files that affect their directory and subdirectories. +Create a `.toak-ignore` file in any directory to specify exclusions. The tool supports nested ignore files that affect their directory and subdirectories. -Example `.code-tokenizer-md-ignore`: +Example `.toak-ignore`: ```plaintext # Ignore specific files secrets.json @@ -154,7 +153,7 @@ This project uses [Bun](https://bun.sh) for development. To contribute: ### Setup ```bash git clone -cd code-tokenizer-md +cd toak bun install ``` diff --git a/package.json b/package.json index 39de939..d298f8c 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,11 @@ { - "name": "code-tokenizer-md", - "version": "1.2.7", + "name": "toak", + "version": "3.0.0", "type": "module", "license": "AGPL-3.0-or-later", - "repository": "https://github.com/geoffsee/code-tokenizer-md.git", + "repository": "https://github.com/seemueller-io/toak.git", "author": "Geoff Seemueller", - "description": "Open-source CLI/Library for turning a git repository's contents into a single well-formatted markdown file.", + "description": "Open-source CLI/Library for tokenizing a git repository into a markdown file.", "exports": { ".": { "import": { @@ -15,7 +15,7 @@ } }, "bin": { - "code-tokenizer-md": "./dist/cli.js" + "toak": "./dist/cli.js" }, "files": [ "dist" diff --git a/src/MarkdownGenerator.ts b/src/MarkdownGenerator.ts index f969839..0cac58f 100644 --- a/src/MarkdownGenerator.ts +++ b/src/MarkdownGenerator.ts @@ -64,7 +64,7 @@ export class MarkdownGenerator { } /** - * Loads and processes .code-tokenizer-md-ignore files recursively from the project directory. + * Loads and processes .toak-ignore files recursively from the project directory. * These files contain patterns for files to exclude from processing. * @async * @returns {Promise} @@ -76,7 +76,7 @@ export class MarkdownGenerator { console.log('Loading ignore patterns...'); } - const ignoreFiles = await glob('**/.code-tokenizer-md-ignore', { + const ignoreFiles = await glob('**/.toak-ignore', { cwd: this.dir, dot: true, absolute: true, diff --git a/code-tokenizer-md.jpg b/toak.jpg similarity index 100% rename from code-tokenizer-md.jpg rename to toak.jpg