From e41532ba82578be64a9b37a9ad0296bb43929db4 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 16:33:48 +0200 Subject: [PATCH 01/17] changed from linear extension search to constant time lookup --- pkg/lines/defaults.go | 61 +++++++++++++++++++++++++++---------------- pkg/lines/lines.go | 13 +++------ 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/pkg/lines/defaults.go b/pkg/lines/defaults.go index 6a8ee37..180b970 100644 --- a/pkg/lines/defaults.go +++ b/pkg/lines/defaults.go @@ -1,33 +1,48 @@ package lines -// DefaultIgnoredDirs returns default directories to ignore. +import "strings" + +// DefaultIgnoredDirs returns slice of default directories to ignore. func DefaultIgnoredDirs() []string { return []string{ "node_modules", "vendor", ".git", "target", } } -// DefaultIgnoredExtensions returns default file extensions to ignore. -func DefaultIgnoredExtensions() []string { - return []string{ - ".exe", ".dll", ".so", ".dylib", - ".zip", ".tar", ".gz", ".bz2", ".xz", - ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", - ".mp3", ".wav", ".flac", ".ogg", ".aac", - ".mp4", ".mkv", ".avi", ".mov", ".wmv", - ".pdf", ".doc", ".docx", ".xls", ".xlsx", - ".icns", ".ttf", ".otf", ".woff", ".woff2", - ".eot", ".svgz", ".uasset", ".plist", - ".url", ".pbxproj", ".sln", - ".vcxproj", ".csproj", ".vcproj", ".tlog", - ".tmp", ".filters", ".idb", ".lock", ".rc", - ".sqlite", ".gdb", ".node", ".rmeta", - ".rlib", ".mcmeta", ".iml", ".map", ".natvis", - ".d", ".dat_old", ".storyboard", ".ilk", ".ppt", - ".pptx", ".odt", ".ods", ".odp", ".odg", ".mca", - ".psd", ".bin", ".jar", ".pdb", ".dox", ".db", - ".schem", ".lnk", ".mod", ".lib", ".o", ".obj", - ".a", ".class", ".pyc", ".pyo", ".whl", ".log", - ".in", ".dat", ".TAG", ".repositories", ".MF", +// DefaultIgnoredExtensions returns set of default file extensions to ignore. +func DefaultIgnoredExtensions() map[string]struct{} { + return makeExtensionSet( + "exe", "dll", "so", "dylib", + "zip", "tar", "gz", "bz2", "xz", + "jpg", "jpeg", + "png", + "gif", "bmp", "webp", "svg", "ico", + "mp3", "wav", "flac", "ogg", "aac", + "mp4", "mkv", "avi", "mov", "wmv", + "pdf", "doc", "docx", "xls", "xlsx", + "icns", "ttf", "otf", "woff", "woff2", + "eot", "svgz", "uasset", "plist", + "url", "pbxproj", "sln", + "vcxproj", "csproj", "vcproj", "tlog", + "tmp", "filters", "idb", "lock", "rc", + "sqlite", "gdb", "node", "rmeta", + "rlib", "mcmeta", "iml", "map", "natvis", + "d", "dat_old", "storyboard", "ilk", "ppt", + "pptx", "odt", "ods", "odp", "odg", "mca", + "psd", "bin", "jar", "pdb", "dox", "db", + "schem", "lnk", "mod", "lib", "o", "obj", + "a", "class", "pyc", "pyo", "whl", "log", + "in", "dat", "TAG", "repositories", "MF", + ) +} + +// makeExtensionSet makes set of file extensions for faster search. +// The extensions are stored in lowercase and prefixed with a dot. +// NOTE: Extensions are prefixed with dot to avoid stripping out the dot for every file. +func makeExtensionSet(items ...string) map[string]struct{} { + set := make(map[string]struct{}, len(items)) + for _, item := range items { + set["."+strings.ToLower(item)] = struct{}{} } + return set } diff --git a/pkg/lines/lines.go b/pkg/lines/lines.go index 4e257dc..163074a 100644 --- a/pkg/lines/lines.go +++ b/pkg/lines/lines.go @@ -18,7 +18,7 @@ type Config struct { // IgnoredDirs are directories to skip during analysis. Defaults to ["node_modules", "vendor", ".git", "target"]. IgnoredDirs []string // IgnoredExtensions are file extensions to skip. Defaults to common binary and media formats. - IgnoredExtensions []string + IgnoredExtensions map[string]struct{} // BufferInitialSize is the initial buffer size for the scanner. Defaults to 64KB. BufferInitialSize int // BufferMaxSize is the maximum buffer size for the scanner. Defaults to 1MB. @@ -72,16 +72,11 @@ func (c *Counter) isIgnoredDir(dirname string) bool { return false } -// isIgnoredExtension checks if a file extension should be ignored. +// isIgnoredExtension checks if a file extension should be ignored for line counting. // The comparison is case-insensitive. func (c *Counter) isIgnoredExtension(ext string) bool { - ext = strings.ToLower(ext) - for _, ignored := range c.config.IgnoredExtensions { - if ext == strings.ToLower(ignored) { - return true - } - } - return false + _, ok := c.config.IgnoredExtensions[strings.ToLower(ext)] + return ok } // Run analyzes the given directory and returns the results. From e4fc78af44be2a4482e9f7a6d6696ce0252c93ce Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 16:41:56 +0200 Subject: [PATCH 02/17] updated version and added version, author, program_name constants --- cmd/lines/cli.go | 5 +++-- cmd/lines/config.go | 5 +++++ cmd/lines/run.go | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 cmd/lines/config.go diff --git a/cmd/lines/cli.go b/cmd/lines/cli.go index c992c3e..1d98d06 100644 --- a/cmd/lines/cli.go +++ b/cmd/lines/cli.go @@ -18,13 +18,14 @@ type cliOptions struct { func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, error) { opts := &cliOptions{} - fs := flag.NewFlagSet("lines", flag.ContinueOnError) + fs := flag.NewFlagSet(PROGRAM_NAME, flag.ContinueOnError) fs.SetOutput(stderr) fs.StringVar(&opts.dir, "dir", ".", "The directory to analyze") fs.BoolVar(&opts.version, "version", false, "Print the version and exit") + fs.BoolVar(&opts.version, "v", false, "Print the version and exit") fs.BoolVar(&opts.help, "help", false, "Print the help message and exit") - fs.BoolVar(&opts.hidden, "hidden", false, "Allows to analize hidden files") + fs.BoolVar(&opts.hidden, "hidden", false, "Allows to analyze hidden files") fs.UintVar(&opts.top, "top", 0, "Print the top N extensions") fs.BoolVar(&opts.noColor, "no-color", false, "Disable color output") fs.BoolVar(&opts.color, "color", false, "Force color output (e.g. when piping)") diff --git a/cmd/lines/config.go b/cmd/lines/config.go new file mode 100644 index 0000000..af3d592 --- /dev/null +++ b/cmd/lines/config.go @@ -0,0 +1,5 @@ +package main + +const PROGRAM_NAME = "lines" +const VERSION = "dev-v1.3.0" +const AUTHOR = "Tymon Wozniak @Moderrek" diff --git a/cmd/lines/run.go b/cmd/lines/run.go index bbd9318..80ccbba 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -22,19 +22,19 @@ func run(stdout, stderr io.Writer, args []string) error { color.NoColor = !useColor if opts.version { - fmt.Fprintln(stdout, "Lines version 1.2.0 created by @Moderrek") + fmt.Printf("%s version %s created by %s\n", PROGRAM_NAME, VERSION, AUTHOR) return nil } if opts.help { - fmt.Fprintln(stderr, "Usage: lines [options]") + fmt.Fprintf(stderr, "Usage: %s [options]\n", PROGRAM_NAME) fs.PrintDefaults() return nil } startTime := time.Now() if isTerminal && !opts.json { - fmt.Fprintf(stderr, "Analyzing.. %s\n\n", opts.dir) + fmt.Fprintf(stderr, "Analyzing ... %s\n\n", opts.dir) } config := lines.Config{ From 15a18fabe7b5bb1050c99eb7bda0482e041fda47 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 17:01:09 +0200 Subject: [PATCH 03/17] explicitly ignoring error --- cmd/lines/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lines/main.go b/cmd/lines/main.go index 2742951..b28acba 100644 --- a/cmd/lines/main.go +++ b/cmd/lines/main.go @@ -7,7 +7,7 @@ import ( func main() { if err := run(os.Stdout, os.Stderr, os.Args); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) + _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } } From eea12264125d887e598d06a52bdeac8116025ed7 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 17:45:09 +0200 Subject: [PATCH 04/17] changed from filepath walk to walkdir, major refactor --- pkg/lines/config.go | 15 ++++ pkg/lines/counter.go | 39 +++++++++ pkg/lines/lines.go | 194 +++++-------------------------------------- pkg/lines/scanner.go | 57 +++++++++++++ pkg/lines/walker.go | 102 +++++++++++++++++++++++ 5 files changed, 236 insertions(+), 171 deletions(-) create mode 100644 pkg/lines/config.go create mode 100644 pkg/lines/counter.go create mode 100644 pkg/lines/scanner.go create mode 100644 pkg/lines/walker.go diff --git a/pkg/lines/config.go b/pkg/lines/config.go new file mode 100644 index 0000000..1b36bf4 --- /dev/null +++ b/pkg/lines/config.go @@ -0,0 +1,15 @@ +package lines + +// Config holds settings for the line counting process. +type Config struct { + // IncludeHidden analyzes hidden files and directories (starting with '.'). + IncludeHidden bool + // IgnoredDirs are directories to skip during analysis. Defaults to ["node_modules", "vendor", ".git", "target"]. + IgnoredDirs []string + // IgnoredExtensions are file extensions to skip. Defaults to common binary and media formats. + IgnoredExtensions map[string]struct{} + // BufferInitialSize is the initial buffer size for the scanner. Defaults to 64KB. + BufferInitialSize int + // BufferMaxSize is the maximum buffer size for the scanner. Defaults to 1MB. + BufferMaxSize int +} diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go new file mode 100644 index 0000000..bfae403 --- /dev/null +++ b/pkg/lines/counter.go @@ -0,0 +1,39 @@ +package lines + +import ( + "sync" + + cmap "github.com/orcaman/concurrent-map/v2" +) + +// TODO: migrate from cmap to stdlib map +// TODO: create workers pool + +// Counter analyzes directories and counts non-blank lines of code. +type Counter struct { + config Config + lines cmap.ConcurrentMap[string, int] + workers sync.WaitGroup +} + +// NewCounter creates a new Counter with the given configuration. +// If IgnoredDirs or IgnoredExtensions are empty, sensible defaults are used. +func NewCounter(config Config) *Counter { + if len(config.IgnoredDirs) == 0 { + config.IgnoredDirs = DefaultIgnoredDirs() + } + if len(config.IgnoredExtensions) == 0 { + config.IgnoredExtensions = DefaultIgnoredExtensions() + } + if config.BufferInitialSize == 0 { + config.BufferInitialSize = 64 * 1024 + } + if config.BufferMaxSize == 0 { + config.BufferMaxSize = 1024 * 1024 + } + + return &Counter{ + config: config, + lines: cmap.New[int](), + } +} diff --git a/pkg/lines/lines.go b/pkg/lines/lines.go index 163074a..206d160 100644 --- a/pkg/lines/lines.go +++ b/pkg/lines/lines.go @@ -1,208 +1,60 @@ package lines import ( - "bufio" "fmt" "os" "path/filepath" "strings" - "sync" - - cmap "github.com/orcaman/concurrent-map/v2" ) -// Config holds settings for the line counting process. -type Config struct { - // IncludeHidden analyzes hidden files and directories (starting with '.'). - IncludeHidden bool - // IgnoredDirs are directories to skip during analysis. Defaults to ["node_modules", "vendor", ".git", "target"]. - IgnoredDirs []string - // IgnoredExtensions are file extensions to skip. Defaults to common binary and media formats. - IgnoredExtensions map[string]struct{} - // BufferInitialSize is the initial buffer size for the scanner. Defaults to 64KB. - BufferInitialSize int - // BufferMaxSize is the maximum buffer size for the scanner. Defaults to 1MB. - BufferMaxSize int -} - -// Represents the results of the line counting process. +// Result represents the results of the line counting process. type Result struct { // LinesByExtension maps file extensions to their total line counts. LinesByExtension map[string]int } -// Counter analyzes directories and counts non-blank lines of code. -// NOTE: Counter is safe for concurrent use and uses goroutines internally. -type Counter struct { - config Config - lines cmap.ConcurrentMap[string, int] - workers sync.WaitGroup -} - -// NewCounter creates a new Counter with the given configuration. -// If IgnoredDirs or IgnoredExtensions are empty, sensible defaults are used. -func NewCounter(config Config) *Counter { - // Use sensible defaults if lists are empty. - if len(config.IgnoredDirs) == 0 { - config.IgnoredDirs = DefaultIgnoredDirs() - } - if len(config.IgnoredExtensions) == 0 { - config.IgnoredExtensions = DefaultIgnoredExtensions() - } - if config.BufferInitialSize == 0 { - config.BufferInitialSize = 64 * 1024 - } - if config.BufferMaxSize == 0 { - config.BufferMaxSize = 1024 * 1024 - } - - return &Counter{ - config: config, - lines: cmap.New[int](), - } -} - -// isIgnoredDir checks if a directory should be ignored. -func (c *Counter) isIgnoredDir(dirname string) bool { - for _, ignored := range c.config.IgnoredDirs { - if dirname == ignored { - return true - } - } - return false -} - -// isIgnoredExtension checks if a file extension should be ignored for line counting. -// The comparison is case-insensitive. -func (c *Counter) isIgnoredExtension(ext string) bool { - _, ok := c.config.IgnoredExtensions[strings.ToLower(ext)] - return ok -} - // Run analyzes the given directory and returns the results. // It recursively walks the directory tree using goroutines for performance. func (c *Counter) Run(dir string) (*Result, error) { if _, err := os.Stat(dir); os.IsNotExist(err) { - return nil, fmt.Errorf("directory '%s' does not exist", dir) + return nil, fmt.Errorf("directory %q does not exist", dir) } c.workers.Add(1) go c.walkDir(dir) c.workers.Wait() - result := &Result{ + return &Result{ LinesByExtension: c.lines.Items(), - } - return result, nil + }, nil } -// walkDir recursively walks the directory tree and counts lines in files. -// It spawns goroutines for each subdirectory to achieve parallel processing. -func (c *Counter) walkDir(dir string) { - defer c.workers.Done() - - visit := func(path string, f os.FileInfo, err error) error { - if err != nil { - // NOTE: Log access errors but continue with other directories. - fmt.Fprintf(os.Stderr, "ERROR: cannot access path %q: %v\n", path, err) - return err - } - if f.IsDir() && path != dir { - dirname := filepath.Base(path) - if !c.config.IncludeHidden && dirname[0] == '.' { - return filepath.SkipDir - } - if c.isIgnoredDir(dirname) { - return filepath.SkipDir - } - c.workers.Add(1) - go c.walkDir(path) - return filepath.SkipDir - } - if f.Mode().IsRegular() { - if c.needToAnalyze(path) { - c.fastLineCounter(path) - } - } - return nil - } - filepath.Walk(dir, visit) -} - -// needToAnalyze determines if a file should be analyzed. -// Returns false if the file is hidden (when IncludeHidden is false), -// has no extension, or has an ignored extension. -func (c *Counter) needToAnalyze(path string) bool { - if !c.config.IncludeHidden && filepath.Base(path)[0] == '.' { - return false - } - extension := filepath.Ext(path) - if len(extension) == 0 { - return false - } - if c.isIgnoredExtension(extension) { - return false - } - return true -} - -// fastLineCounter counts non-blank lines in a file and updates results. -// TODO: Consider caching results for frequently accessed files. -func (c *Counter) fastLineCounter(path string) { - extension := strings.ToLower(filepath.Ext(path)) +// countLinesInFile counts non-blank lines in a file and updates results. +func (c *Counter) countLinesInFile(path string) { + ext := strings.ToLower(filepath.Ext(path)) c.workers.Add(1) + go func() { defer c.workers.Done() - countedLines, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) + + lineCount, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) if err != nil { - // NOTE: Silently skip files with read/encoding issues. - fmt.Fprintf(os.Stderr, "WARNING: failed to count lines in %q: %v\n", path, err) + if _, writeErr := fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", path, err); writeErr != nil { + return + } return } - if countedLines > 0 { - c.lines.Upsert(extension, countedLines, func(exists bool, valueInMap int, newValue int) int { - if exists { - return valueInMap + newValue - } - return newValue - }) - } - }() -} - -// countNonBlankLines reads a file and counts non-blank, non-comment lines. -// Lines starting with '//' or '#' are treated as comments and skipped. -// bufferInitialSize specifies the initial scanner buffer size. -// bufferMaxSize specifies the maximum scanner buffer size. -func countNonBlankLines(path string, bufferInitialSize, bufferMaxSize int) (int, error) { - file, err := os.Open(path) - if err != nil { - return 0, err - } - defer file.Close() - scanner := bufio.NewScanner(file) - buffer := make([]byte, 0, bufferInitialSize) - scanner.Buffer(buffer, bufferMaxSize) - - lineCounter := 0 - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - // Skip empty lines. - if line == "" { - continue - } - // Skip comment lines: //, #, or --. - if strings.HasPrefix(line, "//") || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "--") { - continue + if lineCount <= 0 { + return } - lineCounter++ - } - - if err := scanner.Err(); err != nil { - return 0, err - } - return lineCounter, nil + // Updates result. + c.lines.Upsert(ext, lineCount, func(exists bool, valueInMap int, newValue int) int { + if exists { + return valueInMap + newValue + } + return newValue + }) + }() } diff --git a/pkg/lines/scanner.go b/pkg/lines/scanner.go new file mode 100644 index 0000000..49ab164 --- /dev/null +++ b/pkg/lines/scanner.go @@ -0,0 +1,57 @@ +package lines + +import ( + "bufio" + "bytes" + "fmt" + "os" +) + +// countNonBlankLines reads a file and counts non-blank, non-comment lines. +// Lines starting with '//', '#' or '--' are treated as comments and skipped. +// path specifies the file path to count non-blank lines. +// bufferInitialSize specifies the initial scanner buffer size. +// bufferMaxSize specifies the maximum scanner buffer size. +func countNonBlankLines(path string, bufferInitialSize, bufferMaxSize int) (int, error) { + file, err := os.Open(path) + if err != nil { + return 0, err + } + defer func() { + if err := file.Close(); err != nil { + fmt.Fprintf(os.Stderr, "warn: failed to close file %q: %v\n", path, err) + } + }() + + scanner := bufio.NewScanner(file) + buffer := make([]byte, 0, bufferInitialSize) + scanner.Buffer(buffer, bufferMaxSize) + + commentDoubleSlash := []byte("//") + commentHash := []byte("#") + commentDoubleDash := []byte("--") + + lineCount := 0 + + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + + // Skip empty lines. + if len(line) == 0 { + continue + } + + // Skip comment lines: //, #, or --. + if bytes.HasPrefix(line, commentDoubleSlash) || bytes.HasPrefix(line, commentHash) || bytes.HasPrefix(line, commentDoubleDash) { + continue + } + + lineCount++ + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + return lineCount, nil +} diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go new file mode 100644 index 0000000..d2e0456 --- /dev/null +++ b/pkg/lines/walker.go @@ -0,0 +1,102 @@ +package lines + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// walkDir recursively walks the directory tree and counts lines in files. +// It spawns goroutines for each subdirectory to achieve parallel processing. +func (c *Counter) walkDir(dir string) { + defer c.workers.Done() + + filepath.WalkDir(dir, c.walkFn(dir)) +} + +func (c *Counter) walkFn(root string) fs.WalkDirFunc { + return func(path string, d fs.DirEntry, err error) error { + if err != nil { + return c.handleWalkError(path, err) + } + + if d.IsDir() { + return c.handleDirectory(root, path) + } + + return c.handleFile(path, d) + } +} + +func (c *Counter) handleWalkError(path string, err error) error { + fmt.Fprintf(os.Stderr, "error: cannot read %q: %v\n", path, err) + return err +} + +func (c *Counter) handleDirectory(root string, path string) error { + if path == root { + return nil + } + + name := filepath.Base(path) + + if !c.config.IncludeHidden && strings.HasPrefix(name, ".") { + return filepath.SkipDir + } + + if c.isIgnoredDir(name) { + return filepath.SkipDir + } + + c.workers.Add(1) + go c.walkDir(path) + + return filepath.SkipDir +} + +func (c *Counter) handleFile(path string, d fs.DirEntry) error { + if !d.Type().IsRegular() { + return nil + } + + if c.needToAnalyze(path) { + c.countLinesInFile(path) + } + + return nil +} + +// needToAnalyze determines if a file should be analyzed. +// Returns false if the file is hidden (when IncludeHidden is false), +// has no extension, or has an ignored extension. +func (c *Counter) needToAnalyze(path string) bool { + if !c.config.IncludeHidden && filepath.Base(path)[0] == '.' { + return false + } + + extension := filepath.Ext(path) + if len(extension) == 0 { + return false + } + + return !c.isIgnoredExtension(extension) +} + +// isIgnoredDir checks if a directory should be ignored. +func (c *Counter) isIgnoredDir(dirname string) bool { + for _, ignored := range c.config.IgnoredDirs { + if dirname == ignored { + return true + } + } + return false +} + +// isIgnoredExtension checks if a file extension should be ignored for line counting. +// The comparison is case-insensitive. +func (c *Counter) isIgnoredExtension(ext string) bool { + _, ok := c.config.IgnoredExtensions[strings.ToLower(ext)] + return ok +} From eb3f03d4bf2083cead4a660d611b1df0c32cb97d Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 17:54:56 +0200 Subject: [PATCH 05/17] changed from filepath walk to walkdir, major refactor --- pkg/lines/worker.go | 1 + 1 file changed, 1 insertion(+) create mode 100644 pkg/lines/worker.go diff --git a/pkg/lines/worker.go b/pkg/lines/worker.go new file mode 100644 index 0000000..0da5680 --- /dev/null +++ b/pkg/lines/worker.go @@ -0,0 +1 @@ +package lines From c46422a552336361a121e98b3a34cd3326096625 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 18:45:25 +0200 Subject: [PATCH 06/17] improved file analysis with worker pool and channel for concurrency, removed cmap --- pkg/lines/config.go | 2 ++ pkg/lines/counter.go | 73 ++++++++++++++++++++++++++++++++------------ pkg/lines/lines.go | 60 ------------------------------------ pkg/lines/result.go | 7 +++++ pkg/lines/walker.go | 69 +++++++++++------------------------------ pkg/lines/worker.go | 24 +++++++++++++++ 6 files changed, 105 insertions(+), 130 deletions(-) delete mode 100644 pkg/lines/lines.go create mode 100644 pkg/lines/result.go diff --git a/pkg/lines/config.go b/pkg/lines/config.go index 1b36bf4..b95ee8f 100644 --- a/pkg/lines/config.go +++ b/pkg/lines/config.go @@ -12,4 +12,6 @@ type Config struct { BufferInitialSize int // BufferMaxSize is the maximum buffer size for the scanner. Defaults to 1MB. BufferMaxSize int + // NumWorkers is the number of workers to use for file analysis. + NumWorkers int } diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go index bfae403..5ade130 100644 --- a/pkg/lines/counter.go +++ b/pkg/lines/counter.go @@ -1,39 +1,74 @@ package lines import ( + "fmt" + "os" + "runtime" "sync" - - cmap "github.com/orcaman/concurrent-map/v2" ) -// TODO: migrate from cmap to stdlib map -// TODO: create workers pool - // Counter analyzes directories and counts non-blank lines of code. type Counter struct { - config Config - lines cmap.ConcurrentMap[string, int] - workers sync.WaitGroup + config Config + + linesLock sync.Mutex + lines map[string]int + + workers sync.WaitGroup + filesToAnalyze chan string } // NewCounter creates a new Counter with the given configuration. // If IgnoredDirs or IgnoredExtensions are empty, sensible defaults are used. -func NewCounter(config Config) *Counter { - if len(config.IgnoredDirs) == 0 { - config.IgnoredDirs = DefaultIgnoredDirs() +func NewCounter(cfg Config) *Counter { + if len(cfg.IgnoredDirs) == 0 { + cfg.IgnoredDirs = DefaultIgnoredDirs() } - if len(config.IgnoredExtensions) == 0 { - config.IgnoredExtensions = DefaultIgnoredExtensions() + if len(cfg.IgnoredExtensions) == 0 { + cfg.IgnoredExtensions = DefaultIgnoredExtensions() } - if config.BufferInitialSize == 0 { - config.BufferInitialSize = 64 * 1024 + if cfg.BufferInitialSize == 0 { + cfg.BufferInitialSize = 64 * 1024 } - if config.BufferMaxSize == 0 { - config.BufferMaxSize = 1024 * 1024 + if cfg.BufferMaxSize == 0 { + cfg.BufferMaxSize = 1024 * 1024 + } + if cfg.NumWorkers <= 0 { + cfg.NumWorkers = runtime.NumCPU() * 2 } return &Counter{ - config: config, - lines: cmap.New[int](), + config: cfg, + lines: make(map[string]int), + } +} + +// Run analyzes the given directory and returns the results. +// It recursively walks the directory tree using goroutines for performance. +func (c *Counter) Run(dir string) (*Result, error) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return nil, fmt.Errorf("directory %q does not exist", dir) } + + numWorkers := c.config.NumWorkers + c.filesToAnalyze = make(chan string, numWorkers*4) + + for i := 0; i < numWorkers; i++ { + c.workers.Add(1) + go c.worker() + } + + c.walkDir(dir) + close(c.filesToAnalyze) + c.workers.Wait() + + return &Result{ + LinesByExtension: c.lines, + }, nil +} + +func (c *Counter) addLineCount(ext string, count int) { + c.linesLock.Lock() + c.lines[ext] += count + c.linesLock.Unlock() } diff --git a/pkg/lines/lines.go b/pkg/lines/lines.go deleted file mode 100644 index 206d160..0000000 --- a/pkg/lines/lines.go +++ /dev/null @@ -1,60 +0,0 @@ -package lines - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - -// Result represents the results of the line counting process. -type Result struct { - // LinesByExtension maps file extensions to their total line counts. - LinesByExtension map[string]int -} - -// Run analyzes the given directory and returns the results. -// It recursively walks the directory tree using goroutines for performance. -func (c *Counter) Run(dir string) (*Result, error) { - if _, err := os.Stat(dir); os.IsNotExist(err) { - return nil, fmt.Errorf("directory %q does not exist", dir) - } - - c.workers.Add(1) - go c.walkDir(dir) - c.workers.Wait() - - return &Result{ - LinesByExtension: c.lines.Items(), - }, nil -} - -// countLinesInFile counts non-blank lines in a file and updates results. -func (c *Counter) countLinesInFile(path string) { - ext := strings.ToLower(filepath.Ext(path)) - c.workers.Add(1) - - go func() { - defer c.workers.Done() - - lineCount, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) - if err != nil { - if _, writeErr := fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", path, err); writeErr != nil { - return - } - return - } - - if lineCount <= 0 { - return - } - - // Updates result. - c.lines.Upsert(ext, lineCount, func(exists bool, valueInMap int, newValue int) int { - if exists { - return valueInMap + newValue - } - return newValue - }) - }() -} diff --git a/pkg/lines/result.go b/pkg/lines/result.go new file mode 100644 index 0000000..70e7f4f --- /dev/null +++ b/pkg/lines/result.go @@ -0,0 +1,7 @@ +package lines + +// Result represents the results of the line counting process. +type Result struct { + // LinesByExtension maps file extensions to their total line counts. + LinesByExtension map[string]int +} diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go index d2e0456..5c4bd42 100644 --- a/pkg/lines/walker.go +++ b/pkg/lines/walker.go @@ -1,71 +1,38 @@ package lines import ( - "fmt" "io/fs" - "os" "path/filepath" "strings" ) -// walkDir recursively walks the directory tree and counts lines in files. -// It spawns goroutines for each subdirectory to achieve parallel processing. +// walkDir recursively walks the directory tree and enqueues files for the worker pool to analyze. func (c *Counter) walkDir(dir string) { - defer c.workers.Done() - - filepath.WalkDir(dir, c.walkFn(dir)) -} - -func (c *Counter) walkFn(root string) fs.WalkDirFunc { - return func(path string, d fs.DirEntry, err error) error { + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil { - return c.handleWalkError(path, err) + return nil } if d.IsDir() { - return c.handleDirectory(root, path) + if path == dir { + return nil + } + name := d.Name() + if !c.config.IncludeHidden && strings.HasPrefix(name, ".") { + return filepath.SkipDir + } + if c.isIgnoredDir(name) { + return filepath.SkipDir + } + return nil } - return c.handleFile(path, d) - } -} - -func (c *Counter) handleWalkError(path string, err error) error { - fmt.Fprintf(os.Stderr, "error: cannot read %q: %v\n", path, err) - return err -} - -func (c *Counter) handleDirectory(root string, path string) error { - if path == root { - return nil - } - - name := filepath.Base(path) - - if !c.config.IncludeHidden && strings.HasPrefix(name, ".") { - return filepath.SkipDir - } - - if c.isIgnoredDir(name) { - return filepath.SkipDir - } - - c.workers.Add(1) - go c.walkDir(path) - - return filepath.SkipDir -} + if d.Type().IsRegular() && c.needToAnalyze(path) { + c.filesToAnalyze <- path + } -func (c *Counter) handleFile(path string, d fs.DirEntry) error { - if !d.Type().IsRegular() { return nil - } - - if c.needToAnalyze(path) { - c.countLinesInFile(path) - } - - return nil + }) } // needToAnalyze determines if a file should be analyzed. diff --git a/pkg/lines/worker.go b/pkg/lines/worker.go index 0da5680..af68e12 100644 --- a/pkg/lines/worker.go +++ b/pkg/lines/worker.go @@ -1 +1,25 @@ package lines + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func (c *Counter) worker() { + defer c.workers.Done() + + for path := range c.filesToAnalyze { + lineCount, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) + if err != nil { + fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", path, err) + continue + } + + if lineCount > 0 { + ext := strings.ToLower(filepath.Ext(path)) + c.addLineCount(ext, lineCount) + } + } +} From a2e0c8a8c6085b12cfdd4aea4f3a1803d2c16e09 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 18:45:53 +0200 Subject: [PATCH 07/17] added jobs flag to specify number of concurrent jobs and updated output formatting --- cmd/lines/cli.go | 2 ++ cmd/lines/output.go | 2 +- cmd/lines/run.go | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/lines/cli.go b/cmd/lines/cli.go index 1d98d06..76ce0c0 100644 --- a/cmd/lines/cli.go +++ b/cmd/lines/cli.go @@ -14,6 +14,7 @@ type cliOptions struct { noColor bool color bool json bool + jobs uint } func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, error) { @@ -27,6 +28,7 @@ func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, er fs.BoolVar(&opts.help, "help", false, "Print the help message and exit") fs.BoolVar(&opts.hidden, "hidden", false, "Allows to analyze hidden files") fs.UintVar(&opts.top, "top", 0, "Print the top N extensions") + fs.UintVar(&opts.jobs, "jobs", 0, "Specifies the number of jobs") fs.BoolVar(&opts.noColor, "no-color", false, "Disable color output") fs.BoolVar(&opts.color, "color", false, "Force color output (e.g. when piping)") fs.BoolVar(&opts.json, "json", false, "Output results in JSON format") diff --git a/cmd/lines/output.go b/cmd/lines/output.go index 42a6a41..826bff7 100644 --- a/cmd/lines/output.go +++ b/cmd/lines/output.go @@ -42,7 +42,7 @@ func printHumanOutput(w io.Writer, result *lines.Result, opts *cliOptions) { } extColor.Fprintf(w, "%s", key) - fmt.Fprint(w, " ") // Separator + fmt.Fprint(w, "\t") linesColor.Fprintf(w, "%d\n", linesCount) } } diff --git a/cmd/lines/run.go b/cmd/lines/run.go index 80ccbba..3140f8d 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -34,11 +34,12 @@ func run(stdout, stderr io.Writer, args []string) error { startTime := time.Now() if isTerminal && !opts.json { - fmt.Fprintf(stderr, "Analyzing ... %s\n\n", opts.dir) + fmt.Fprintf(stderr, "Analyzing ...\n") } config := lines.Config{ IncludeHidden: opts.hidden, + NumWorkers: int(opts.jobs), } counter := lines.NewCounter(config) result, err := counter.Run(opts.dir) From 83f0e8274c6cc55fffdd5131bc54065b4451b51d Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 18:46:16 +0200 Subject: [PATCH 08/17] removed concurrent-map dependency --- go.mod | 2 -- go.sum | 2 -- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index 02edd52..28b2e79 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/moderrek/lines go 1.22.4 -require github.com/orcaman/concurrent-map/v2 v2.0.1 - require ( github.com/fatih/color v1.17.0 github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index 91faaae..4ddf511 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= -github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= From 7a49344dccd92052c496296793417bf1bed57b2f Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 19:12:02 +0200 Subject: [PATCH 09/17] needToAnalyze accepts filename and improved output formatting --- cmd/lines/output.go | 3 +++ cmd/lines/run.go | 2 +- pkg/lines/walker.go | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/lines/output.go b/cmd/lines/output.go index 826bff7..8469dba 100644 --- a/cmd/lines/output.go +++ b/cmd/lines/output.go @@ -21,10 +21,12 @@ func printJSONOutput(w io.Writer, result *lines.Result) error { func printHumanOutput(w io.Writer, result *lines.Result, opts *cliOptions) { lineMap := result.LinesByExtension + sortedKeys := make([]string, 0, len(lineMap)) for key := range lineMap { sortedKeys = append(sortedKeys, key) } + sort.Slice(sortedKeys, func(i, j int) bool { return lineMap[sortedKeys[i]] > lineMap[sortedKeys[j]] }) @@ -36,6 +38,7 @@ func printHumanOutput(w io.Writer, result *lines.Result, opts *cliOptions) { if opts.top > 0 && uint(i) >= opts.top { break } + linesCount := lineMap[key] if linesCount == 0 { continue diff --git a/cmd/lines/run.go b/cmd/lines/run.go index 3140f8d..d429d4d 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -27,7 +27,7 @@ func run(stdout, stderr io.Writer, args []string) error { } if opts.help { - fmt.Fprintf(stderr, "Usage: %s [options]\n", PROGRAM_NAME) + fmt.Printf("Usage: %s [options]\n", PROGRAM_NAME) fs.PrintDefaults() return nil } diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go index 5c4bd42..f6b66a0 100644 --- a/pkg/lines/walker.go +++ b/pkg/lines/walker.go @@ -27,7 +27,7 @@ func (c *Counter) walkDir(dir string) { return nil } - if d.Type().IsRegular() && c.needToAnalyze(path) { + if d.Type().IsRegular() && c.needToAnalyze(path, d.Name()) { c.filesToAnalyze <- path } @@ -38,8 +38,8 @@ func (c *Counter) walkDir(dir string) { // needToAnalyze determines if a file should be analyzed. // Returns false if the file is hidden (when IncludeHidden is false), // has no extension, or has an ignored extension. -func (c *Counter) needToAnalyze(path string) bool { - if !c.config.IncludeHidden && filepath.Base(path)[0] == '.' { +func (c *Counter) needToAnalyze(path, filename string) bool { + if !c.config.IncludeHidden && filename[0] == '.' { return false } From cbaf2c72823962f5364bebd995a07c06fdc159cb Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 20:49:44 +0200 Subject: [PATCH 10/17] added progress reporting and atomic counters for files found and processed --- cmd/lines/output.go | 4 +-- cmd/lines/run.go | 61 ++++++++++++++++++++++++++++++++++++-------- pkg/lines/counter.go | 4 +++ pkg/lines/walker.go | 1 + pkg/lines/worker.go | 1 + 5 files changed, 58 insertions(+), 13 deletions(-) diff --git a/cmd/lines/output.go b/cmd/lines/output.go index 8469dba..18c3fb9 100644 --- a/cmd/lines/output.go +++ b/cmd/lines/output.go @@ -15,8 +15,8 @@ func printJSONOutput(w io.Writer, result *lines.Result) error { if err != nil { return fmt.Errorf("error generating JSON: %w", err) } - fmt.Fprintln(w, string(jsonOutput)) - return nil + _, err = fmt.Fprintln(w, string(jsonOutput)) + return err } func printHumanOutput(w io.Writer, result *lines.Result, opts *cliOptions) { diff --git a/cmd/lines/run.go b/cmd/lines/run.go index d429d4d..c6996ca 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -17,8 +17,10 @@ func run(stdout, stderr io.Writer, args []string) error { return err } - isTerminal := isatty.IsTerminal(os.Stdout.Fd()) - useColor := (isTerminal || opts.color) && !opts.noColor + isStdoutTerminal := isatty.IsTerminal(os.Stdout.Fd()) + isStderrTerminal := isatty.IsTerminal(os.Stderr.Fd()) + + useColor := (isStdoutTerminal || opts.color) && !opts.noColor color.NoColor = !useColor if opts.version { @@ -27,35 +29,72 @@ func run(stdout, stderr io.Writer, args []string) error { } if opts.help { - fmt.Printf("Usage: %s [options]\n", PROGRAM_NAME) + fmt.Fprintf(stdout, "Usage: %s [options]\n", PROGRAM_NAME) + fs.SetOutput(stdout) fs.PrintDefaults() return nil } - startTime := time.Now() - if isTerminal && !opts.json { - fmt.Fprintf(stderr, "Analyzing ...\n") - } - config := lines.Config{ IncludeHidden: opts.hidden, NumWorkers: int(opts.jobs), } counter := lines.NewCounter(config) + + stopProgress := make(chan struct{}) + defer close(stopProgress) + + startTime := time.Now() + + showProgress := isStderrTerminal && !opts.json + if showProgress { + go startProgressReporter(stderr, stopProgress, startTime, counter) + } + result, err := counter.Run(opts.dir) if err != nil { return err } + if showProgress { + stopProgress <- struct{}{} + reportProgress(stderr, startTime, counter) + fmt.Fprintf(stderr, "\n") + } + if opts.json { return printJSONOutput(stdout, result) } printHumanOutput(stdout, result, opts) - if isTerminal { - color.New(color.FgGreen).Fprintf(stderr, "\nTime taken: %v to analyze files\n", time.Since(startTime)) + return nil +} + +func startProgressReporter(w io.Writer, stop chan struct{}, startTime time.Time, counter *lines.Counter) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-stop: + return + case <-ticker.C: + reportProgress(w, startTime, counter) + } } +} - return nil +func reportProgress(w io.Writer, startTime time.Time, counter *lines.Counter) { + processed := counter.FilesProcessed.Load() + found := counter.FilesFound.Load() + + elapsed := time.Since(startTime) + + inQueue := found - processed + if inQueue < 0 { + inQueue = 0 + } + + fmt.Fprintf(w, "\r\033[KProcessed: %d | In Queue: %d | Elapsed: %v", processed, inQueue, elapsed) } diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go index 5ade130..2285770 100644 --- a/pkg/lines/counter.go +++ b/pkg/lines/counter.go @@ -5,6 +5,7 @@ import ( "os" "runtime" "sync" + "sync/atomic" ) // Counter analyzes directories and counts non-blank lines of code. @@ -16,6 +17,9 @@ type Counter struct { workers sync.WaitGroup filesToAnalyze chan string + + FilesFound atomic.Int64 + FilesProcessed atomic.Int64 } // NewCounter creates a new Counter with the given configuration. diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go index f6b66a0..cac2cb9 100644 --- a/pkg/lines/walker.go +++ b/pkg/lines/walker.go @@ -28,6 +28,7 @@ func (c *Counter) walkDir(dir string) { } if d.Type().IsRegular() && c.needToAnalyze(path, d.Name()) { + c.FilesFound.Add(1) c.filesToAnalyze <- path } diff --git a/pkg/lines/worker.go b/pkg/lines/worker.go index af68e12..b18a53d 100644 --- a/pkg/lines/worker.go +++ b/pkg/lines/worker.go @@ -12,6 +12,7 @@ func (c *Counter) worker() { for path := range c.filesToAnalyze { lineCount, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) + c.FilesProcessed.Add(1) if err != nil { fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", path, err) continue From 02d63c0de93e5f92c3ffb9db06404446674db51d Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sat, 4 Jul 2026 20:57:07 +0200 Subject: [PATCH 11/17] refactor scanner to use bufio.Reader for improved line reading and handling long lines --- pkg/lines/scanner.go | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/pkg/lines/scanner.go b/pkg/lines/scanner.go index 49ab164..f22db00 100644 --- a/pkg/lines/scanner.go +++ b/pkg/lines/scanner.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "fmt" + "io" "os" ) @@ -23,35 +24,54 @@ func countNonBlankLines(path string, bufferInitialSize, bufferMaxSize int) (int, } }() - scanner := bufio.NewScanner(file) - buffer := make([]byte, 0, bufferInitialSize) - scanner.Buffer(buffer, bufferMaxSize) + var r *bufio.Reader + if bufferInitialSize > 0 { + r = bufio.NewReaderSize(file, bufferInitialSize) + } else { + r = bufio.NewReader(file) + } commentDoubleSlash := []byte("//") commentHash := []byte("#") commentDoubleDash := []byte("--") lineCount := 0 + isInsideLongLine := false + + for { + line, isPrefix, err := r.ReadLine() + if err != nil { + if err == io.EOF { + break + } + return 0, err + } - for scanner.Scan() { - line := bytes.TrimSpace(scanner.Bytes()) + if isInsideLongLine { + if !isPrefix { + isInsideLongLine = false + } + continue + } + + if isPrefix { + isInsideLongLine = true + } + + cleaned := bytes.TrimSpace(line) // Skip empty lines. - if len(line) == 0 { + if len(cleaned) == 0 { continue } // Skip comment lines: //, #, or --. - if bytes.HasPrefix(line, commentDoubleSlash) || bytes.HasPrefix(line, commentHash) || bytes.HasPrefix(line, commentDoubleDash) { + if bytes.HasPrefix(cleaned, commentDoubleSlash) || bytes.HasPrefix(cleaned, commentHash) || bytes.HasPrefix(cleaned, commentDoubleDash) { continue } lineCount++ } - if err := scanner.Err(); err != nil { - return 0, err - } - return lineCount, nil } From 27ffd8d50cca47bb35f9a1a225ab11bf0915d959 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Sun, 5 Jul 2026 09:27:02 +0200 Subject: [PATCH 12/17] improved file analysis logic and added verbose logging --- cmd/lines/cli.go | 2 + cmd/lines/run.go | 1 + pkg/lines/{scanner.go => analysis.go} | 8 +-- pkg/lines/config.go | 10 ++-- pkg/lines/counter.go | 75 +++++++++++++++++++-------- pkg/lines/defaults.go | 13 +++-- pkg/lines/log.go | 12 +++++ pkg/lines/walker.go | 44 +++++++++------- pkg/lines/worker.go | 8 +-- 9 files changed, 115 insertions(+), 58 deletions(-) rename pkg/lines/{scanner.go => analysis.go} (84%) create mode 100644 pkg/lines/log.go diff --git a/cmd/lines/cli.go b/cmd/lines/cli.go index 76ce0c0..6eca4ff 100644 --- a/cmd/lines/cli.go +++ b/cmd/lines/cli.go @@ -15,6 +15,7 @@ type cliOptions struct { color bool json bool jobs uint + verbose bool } func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, error) { @@ -32,6 +33,7 @@ func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, er fs.BoolVar(&opts.noColor, "no-color", false, "Disable color output") fs.BoolVar(&opts.color, "color", false, "Force color output (e.g. when piping)") fs.BoolVar(&opts.json, "json", false, "Output results in JSON format") + fs.BoolVar(&opts.verbose, "verbose", false, "Verbose output") err := fs.Parse(args[1:]) if err != nil { diff --git a/cmd/lines/run.go b/cmd/lines/run.go index c6996ca..40ffa4b 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -36,6 +36,7 @@ func run(stdout, stderr io.Writer, args []string) error { } config := lines.Config{ + Verbose: opts.verbose, IncludeHidden: opts.hidden, NumWorkers: int(opts.jobs), } diff --git a/pkg/lines/scanner.go b/pkg/lines/analysis.go similarity index 84% rename from pkg/lines/scanner.go rename to pkg/lines/analysis.go index f22db00..d296374 100644 --- a/pkg/lines/scanner.go +++ b/pkg/lines/analysis.go @@ -8,12 +8,12 @@ import ( "os" ) -// countNonBlankLines reads a file and counts non-blank, non-comment lines. +// analyzeFile reads a file and counts non-blank, non-comment lines. // Lines starting with '//', '#' or '--' are treated as comments and skipped. // path specifies the file path to count non-blank lines. // bufferInitialSize specifies the initial scanner buffer size. // bufferMaxSize specifies the maximum scanner buffer size. -func countNonBlankLines(path string, bufferInitialSize, bufferMaxSize int) (int, error) { +func analyzeFile(path string, readerInitialBufferSize int) (int, error) { file, err := os.Open(path) if err != nil { return 0, err @@ -25,8 +25,8 @@ func countNonBlankLines(path string, bufferInitialSize, bufferMaxSize int) (int, }() var r *bufio.Reader - if bufferInitialSize > 0 { - r = bufio.NewReaderSize(file, bufferInitialSize) + if readerInitialBufferSize > 0 { + r = bufio.NewReaderSize(file, readerInitialBufferSize) } else { r = bufio.NewReader(file) } diff --git a/pkg/lines/config.go b/pkg/lines/config.go index b95ee8f..ab5ddc6 100644 --- a/pkg/lines/config.go +++ b/pkg/lines/config.go @@ -5,13 +5,13 @@ type Config struct { // IncludeHidden analyzes hidden files and directories (starting with '.'). IncludeHidden bool // IgnoredDirs are directories to skip during analysis. Defaults to ["node_modules", "vendor", ".git", "target"]. - IgnoredDirs []string + IgnoredDirs map[string]struct{} // IgnoredExtensions are file extensions to skip. Defaults to common binary and media formats. IgnoredExtensions map[string]struct{} - // BufferInitialSize is the initial buffer size for the scanner. Defaults to 64KB. - BufferInitialSize int - // BufferMaxSize is the maximum buffer size for the scanner. Defaults to 1MB. - BufferMaxSize int + // ReaderInitialBufferSize is the initial buffer size for the scanner. + ReaderInitialBufferSize int // NumWorkers is the number of workers to use for file analysis. NumWorkers int + // Verbose enables logging while file analysis. + Verbose bool } diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go index 2285770..47844c7 100644 --- a/pkg/lines/counter.go +++ b/pkg/lines/counter.go @@ -3,6 +3,7 @@ package lines import ( "fmt" "os" + "path/filepath" "runtime" "sync" "sync/atomic" @@ -10,7 +11,7 @@ import ( // Counter analyzes directories and counts non-blank lines of code. type Counter struct { - config Config + Config Config linesLock sync.Mutex lines map[string]int @@ -24,50 +25,70 @@ type Counter struct { // NewCounter creates a new Counter with the given configuration. // If IgnoredDirs or IgnoredExtensions are empty, sensible defaults are used. -func NewCounter(cfg Config) *Counter { - if len(cfg.IgnoredDirs) == 0 { - cfg.IgnoredDirs = DefaultIgnoredDirs() - } - if len(cfg.IgnoredExtensions) == 0 { - cfg.IgnoredExtensions = DefaultIgnoredExtensions() +func NewCounter(config Config) *Counter { + return &Counter{ + Config: configWithDefaults(config), + linesLock: sync.Mutex{}, + lines: make(map[string]int), + workers: sync.WaitGroup{}, + filesToAnalyze: nil, + FilesFound: atomic.Int64{}, + FilesProcessed: atomic.Int64{}, } - if cfg.BufferInitialSize == 0 { - cfg.BufferInitialSize = 64 * 1024 +} + +func configWithDefaults(config Config) Config { + if len(config.IgnoredDirs) == 0 { + config.IgnoredDirs = DefaultIgnoredDirs() } - if cfg.BufferMaxSize == 0 { - cfg.BufferMaxSize = 1024 * 1024 + if len(config.IgnoredExtensions) == 0 { + config.IgnoredExtensions = DefaultIgnoredExtensions() } - if cfg.NumWorkers <= 0 { - cfg.NumWorkers = runtime.NumCPU() * 2 + if config.ReaderInitialBufferSize == 0 { + config.ReaderInitialBufferSize = 64 * 1024 } - - return &Counter{ - config: cfg, - lines: make(map[string]int), + if config.NumWorkers <= 0 { + config.NumWorkers = runtime.NumCPU() * 2 } + return config } // Run analyzes the given directory and returns the results. // It recursively walks the directory tree using goroutines for performance. func (c *Counter) Run(dir string) (*Result, error) { + c.reset() + if _, err := os.Stat(dir); os.IsNotExist(err) { return nil, fmt.Errorf("directory %q does not exist", dir) } - numWorkers := c.config.NumWorkers - c.filesToAnalyze = make(chan string, numWorkers*4) + numWorkers := c.Config.NumWorkers + maximumWaitingWork := numWorkers * 4 + c.filesToAnalyze = make(chan string, maximumWaitingWork) + c.logVerbosef("creating %d workers with work queue of capacity %d", numWorkers, maximumWaitingWork) for i := 0; i < numWorkers; i++ { c.workers.Add(1) - go c.worker() + go c.analyzeFilesWorker() } - c.walkDir(dir) + c.logVerbosef("starting analysis at: %q", filepath.ToSlash(dir)) + err := c.walkDir(dir) + if err != nil { + return nil, err + } close(c.filesToAnalyze) c.workers.Wait() + // Makes copy of result. + c.logVerbosef("coping result") + linesByExt := make(map[string]int) + for ext, count := range c.lines { + linesByExt[ext] = count + } + return &Result{ - LinesByExtension: c.lines, + LinesByExtension: linesByExt, }, nil } @@ -76,3 +97,13 @@ func (c *Counter) addLineCount(ext string, count int) { c.lines[ext] += count c.linesLock.Unlock() } + +func (c *Counter) reset() { + c.workers = sync.WaitGroup{} + c.FilesFound.Store(0) + c.FilesProcessed.Store(0) + + c.linesLock.Lock() + c.lines = make(map[string]int) + c.linesLock.Unlock() +} diff --git a/pkg/lines/defaults.go b/pkg/lines/defaults.go index 180b970..82ce3c1 100644 --- a/pkg/lines/defaults.go +++ b/pkg/lines/defaults.go @@ -3,19 +3,22 @@ package lines import "strings" // DefaultIgnoredDirs returns slice of default directories to ignore. -func DefaultIgnoredDirs() []string { - return []string{ - "node_modules", "vendor", ".git", "target", +func DefaultIgnoredDirs() map[string]struct{} { + return map[string]struct{}{ + "node_modules": {}, + "vendor": {}, + ".git": {}, + "target": {}, } } // DefaultIgnoredExtensions returns set of default file extensions to ignore. func DefaultIgnoredExtensions() map[string]struct{} { return makeExtensionSet( - "exe", "dll", "so", "dylib", + "exe", "dll", "so", "dylib", "msi", "mui", "mun", "zip", "tar", "gz", "bz2", "xz", "jpg", "jpeg", - "png", + "png", "dng", "heic", "gif", "bmp", "webp", "svg", "ico", "mp3", "wav", "flac", "ogg", "aac", "mp4", "mkv", "avi", "mov", "wmv", diff --git a/pkg/lines/log.go b/pkg/lines/log.go new file mode 100644 index 0000000..3ac0576 --- /dev/null +++ b/pkg/lines/log.go @@ -0,0 +1,12 @@ +package lines + +import ( + "fmt" + "os" +) + +func (c *Counter) logVerbosef(format string, v ...any) { + if c.Config.Verbose { + fmt.Fprintf(os.Stderr, "info: "+format+"\n", v...) + } +} diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go index cac2cb9..9ce3778 100644 --- a/pkg/lines/walker.go +++ b/pkg/lines/walker.go @@ -1,15 +1,18 @@ package lines import ( + "fmt" "io/fs" + "os" "path/filepath" "strings" ) -// walkDir recursively walks the directory tree and enqueues files for the worker pool to analyze. -func (c *Counter) walkDir(dir string) { - _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { +// walkDir recursively walks the directory tree and enqueues files for the analyzeFilesWorker pool to analyze. +func (c *Counter) walkDir(dir string) error { + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil { + fmt.Fprintf(os.Stderr, "warn: failed to access path %q: %v\n", filepath.ToSlash(path), err) return nil } @@ -18,53 +21,56 @@ func (c *Counter) walkDir(dir string) { return nil } name := d.Name() - if !c.config.IncludeHidden && strings.HasPrefix(name, ".") { + if !c.Config.IncludeHidden && strings.HasPrefix(name, ".") { + c.logVerbosef("skipping hidden directory: %q", filepath.ToSlash(path)) return filepath.SkipDir } if c.isIgnoredDir(name) { + c.logVerbosef("skipping ignored directory: %q", filepath.ToSlash(path)) return filepath.SkipDir } return nil } - if d.Type().IsRegular() && c.needToAnalyze(path, d.Name()) { + if d.Type().IsRegular() && c.shouldAnalyzeFile(path, d.Name()) { + c.logVerbosef("found file to analyze: %q", filepath.ToSlash(path)) c.FilesFound.Add(1) c.filesToAnalyze <- path + } else { + c.logVerbosef("skipping file: %q", filepath.ToSlash(path)) } return nil }) + return err } -// needToAnalyze determines if a file should be analyzed. +// shouldAnalyzeFile determines if a file should be analyzed. // Returns false if the file is hidden (when IncludeHidden is false), // has no extension, or has an ignored extension. -func (c *Counter) needToAnalyze(path, filename string) bool { - if !c.config.IncludeHidden && filename[0] == '.' { +func (c *Counter) shouldAnalyzeFile(path, filename string) bool { + if !c.Config.IncludeHidden && strings.HasPrefix(filename, ".") { return false } - extension := filepath.Ext(path) - if len(extension) == 0 { + ext := filepath.Ext(path) + if len(ext) == 0 { + // probably its binary file return false } - return !c.isIgnoredExtension(extension) + return !c.isIgnoredExtension(ext) } // isIgnoredDir checks if a directory should be ignored. func (c *Counter) isIgnoredDir(dirname string) bool { - for _, ignored := range c.config.IgnoredDirs { - if dirname == ignored { - return true - } - } - return false + _, ok := c.Config.IgnoredDirs[dirname] + return ok } // isIgnoredExtension checks if a file extension should be ignored for line counting. -// The comparison is case-insensitive. +// The comparison is case-insensitive. Extension should begin with dot. func (c *Counter) isIgnoredExtension(ext string) bool { - _, ok := c.config.IgnoredExtensions[strings.ToLower(ext)] + _, ok := c.Config.IgnoredExtensions[strings.ToLower(ext)] return ok } diff --git a/pkg/lines/worker.go b/pkg/lines/worker.go index b18a53d..eb01a9c 100644 --- a/pkg/lines/worker.go +++ b/pkg/lines/worker.go @@ -7,20 +7,22 @@ import ( "strings" ) -func (c *Counter) worker() { +func (c *Counter) analyzeFilesWorker() { defer c.workers.Done() for path := range c.filesToAnalyze { - lineCount, err := countNonBlankLines(path, c.config.BufferInitialSize, c.config.BufferMaxSize) + lineCount, err := analyzeFile(path, c.Config.ReaderInitialBufferSize) c.FilesProcessed.Add(1) if err != nil { - fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", path, err) + fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", filepath.ToSlash(path), err) continue } if lineCount > 0 { ext := strings.ToLower(filepath.Ext(path)) c.addLineCount(ext, lineCount) + c.logVerbosef("file %q had %d lines", filepath.ToSlash(path), lineCount) + } } } From 6bc66e31e764e2937efdc56ff3b83e0925ad0399 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Mon, 6 Jul 2026 13:38:30 +0200 Subject: [PATCH 13/17] updated README --- README.md | 54 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 1317e0d..0291e9a 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,8 @@ -A concurrent, non-blank line counter for source code directories, written in GO. - +A concurrent non-blank line counter for source code directories, written in Go. It recursively walks a directory, concurrently analyzes files, and reports the number of non-blank lines of code, grouped by file extension. - The tool is designed for performance, utilizing goroutines to process files in parallel. ## Installation @@ -37,20 +35,24 @@ The `lines` command accepts the following flags: Usage: lines [options] Options: - -dir string - The directory to analyze (default ".") - -hidden - Include hidden files and directories in the analysis - -top uint - Show only the top N extensions by line count + -color + Force color output (e.g. when piping) -no-color - Disable colorized output + Disable color output + -help + Print the help message + -hidden + Allows to analyze hidden files + -jobs uint + Specifies the number of jobs -json Output results in JSON format + -top uint + Print the top N extensions + -verbose + Verbose output -version - Print version information and exit - -help - Show this help message and exit + Print the version ``` ### Example @@ -58,23 +60,23 @@ Options: To analyze the directory `~/projects/my-app` and display the top 5 extensions: ```shell -lines --dir ~/projects/my-app --top 5 +lines --top 5 ~/projects/my-app ``` To get the output in JSON format, which can be piped to other tools like `jq`: ```shell -lines --dir ~/projects/my-app --json +lines --json ~/projects/my-app ``` Example output (`--json`): ```json { - ".css": 1122, - ".go": 15230, - ".html": 4357, - ".js": 8828, - ".mod": 4980 + ".css": 1122, + ".go": 15230, + ".html": 4357, + ".js": 8828, + ".mod": 4980 } ``` @@ -126,8 +128,14 @@ You can customize which directories and file extensions to ignore: ```go config := lines.Config{ IncludeHidden: false, - IgnoredDirs: []string{"node_modules", "vendor", ".git", "target", "dist"}, - IgnoredExtensions: []string{".exe", ".dll", ".jpg", ".png"}, + IgnoredDirs: map[string]struct{}{ + "node_modules": {}, + ".git": {}, + }, + IgnoredExtensions: map[string]struct{}{ + ".exe": {}, + ".env": {}, + }, } counter := lines.NewCounter(config) result, err := counter.Run("./src") @@ -154,4 +162,4 @@ If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensi ## License This project is licensed under the MIT License. -See the [LICENSE](LICENSE) file for details. +See the [LICENSE](./LICENSE) file for details. From fa33cf6d353668c9dd3a2a604104b5286ccd6ab8 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Mon, 6 Jul 2026 22:18:10 +0200 Subject: [PATCH 14/17] simplified analysis and worker --- pkg/lines/analysis.go | 74 ++++++++++++++++++++++++++----------------- pkg/lines/worker.go | 14 +++++--- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/pkg/lines/analysis.go b/pkg/lines/analysis.go index d296374..64c9d95 100644 --- a/pkg/lines/analysis.go +++ b/pkg/lines/analysis.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" ) // analyzeFile reads a file and counts non-blank, non-comment lines. @@ -16,30 +17,21 @@ import ( func analyzeFile(path string, readerInitialBufferSize int) (int, error) { file, err := os.Open(path) if err != nil { - return 0, err + return 0, fmt.Errorf("failed to analyze %q: %v", filepath.ToSlash(path), err) } + defer func() { if err := file.Close(); err != nil { - fmt.Fprintf(os.Stderr, "warn: failed to close file %q: %v\n", path, err) + fmt.Fprintf(os.Stderr, "warn: failed to close file %q: %v\n", filepath.ToSlash(path), err) } }() - var r *bufio.Reader - if readerInitialBufferSize > 0 { - r = bufio.NewReaderSize(file, readerInitialBufferSize) - } else { - r = bufio.NewReader(file) - } - - commentDoubleSlash := []byte("//") - commentHash := []byte("#") - commentDoubleDash := []byte("--") - + reader := newReader(file, readerInitialBufferSize) lineCount := 0 isInsideLongLine := false for { - line, isPrefix, err := r.ReadLine() + line, err := readLine(reader, path, &isInsideLongLine) if err != nil { if err == io.EOF { break @@ -47,31 +39,55 @@ func analyzeFile(path string, readerInitialBufferSize int) (int, error) { return 0, err } - if isInsideLongLine { - if !isPrefix { - isInsideLongLine = false - } + if len(line) == 0 { continue } - if isPrefix { - isInsideLongLine = true + if isCommentLine(line) { + continue } - cleaned := bytes.TrimSpace(line) + lineCount++ + } - // Skip empty lines. - if len(cleaned) == 0 { - continue + return lineCount, nil +} + +func readLine(reader *bufio.Reader, path string, isInsideLongLine *bool) ([]byte, error) { + line, isPrefix, err := reader.ReadLine() + if err != nil { + if err == io.EOF { + return nil, err } + return nil, fmt.Errorf("failed to analyze %q: %v", filepath.ToSlash(path), err) + } - // Skip comment lines: //, #, or --. - if bytes.HasPrefix(cleaned, commentDoubleSlash) || bytes.HasPrefix(cleaned, commentHash) || bytes.HasPrefix(cleaned, commentDoubleDash) { - continue + if *isInsideLongLine { + if !isPrefix { + *isInsideLongLine = false } + return nil, nil + } - lineCount++ + if isPrefix { + *isInsideLongLine = true } - return lineCount, nil + trimmedLine := bytes.TrimSpace(line) + return trimmedLine, nil +} + +func isCommentLine(line []byte) bool { + doubleSlash := []byte("//") + hash := []byte("#") + doubleDash := []byte("--") + + return bytes.HasPrefix(line, doubleSlash) || bytes.HasPrefix(line, hash) || bytes.HasPrefix(line, doubleDash) +} + +func newReader(file *os.File, initialBufferSize int) *bufio.Reader { + if initialBufferSize > 0 { + return bufio.NewReaderSize(file, initialBufferSize) + } + return bufio.NewReader(file) } diff --git a/pkg/lines/worker.go b/pkg/lines/worker.go index eb01a9c..cdd1942 100644 --- a/pkg/lines/worker.go +++ b/pkg/lines/worker.go @@ -13,16 +13,20 @@ func (c *Counter) analyzeFilesWorker() { for path := range c.filesToAnalyze { lineCount, err := analyzeFile(path, c.Config.ReaderInitialBufferSize) c.FilesProcessed.Add(1) + if err != nil { + // logs the error and continues processing other files. fmt.Fprintf(os.Stderr, "warn: failed to count lines in %q: %v\n", filepath.ToSlash(path), err) continue } - if lineCount > 0 { - ext := strings.ToLower(filepath.Ext(path)) - c.addLineCount(ext, lineCount) - c.logVerbosef("file %q had %d lines", filepath.ToSlash(path), lineCount) - + if lineCount <= 0 { + // file is empty. + continue } + + ext := strings.ToLower(filepath.Ext(path)) + c.addLineCount(ext, lineCount) + c.logVerbosef("file %q had %d lines", filepath.ToSlash(path), lineCount) } } From a8b524267a4ee78fe0e8e86d85db976b299ac0e3 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Mon, 6 Jul 2026 22:20:19 +0200 Subject: [PATCH 15/17] improved error handling in analysis --- pkg/lines/analysis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lines/analysis.go b/pkg/lines/analysis.go index 64c9d95..ed33ec4 100644 --- a/pkg/lines/analysis.go +++ b/pkg/lines/analysis.go @@ -36,7 +36,7 @@ func analyzeFile(path string, readerInitialBufferSize int) (int, error) { if err == io.EOF { break } - return 0, err + return 0, fmt.Errorf("failed to analyze %q: readLine: %v\n", filepath.ToSlash(path), err) } if len(line) == 0 { From 3e8893bdd27958c80d9597a6af2ba1355761bb17 Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Wed, 29 Jul 2026 18:58:19 +0200 Subject: [PATCH 16/17] refactor: standardize naming conventions and improve README instructions --- README.md | 6 ++-- cmd/lines/cli.go | 2 +- cmd/lines/config.go | 6 ++-- cmd/lines/output.go | 2 +- cmd/lines/run.go | 13 ++++--- go.mod | 2 +- pkg/lines/config.go | 4 +++ pkg/lines/counter.go | 85 ++++++++++++++++++++++++++++++++++++-------- pkg/lines/log.go | 8 +++-- pkg/lines/result.go | 16 +++++++++ pkg/lines/walker.go | 5 +++ 11 files changed, 120 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 0291e9a..2737478 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The tool is designed for performance, utilizing goroutines to process files in p To install the `lines` command-line tool, ensure you have [Go](https://go.dev/doc/install) installed and configured, then run: ```shell -go install github.com/moderrek/lines/cmd/lines@latest +go install github.com/Moderrek/lines/cmd/lines@latest ``` This will download the source, compile it, and place the `lines` binary in your Go bin directory (`$GOPATH/bin` or `$HOME/go/in`). @@ -86,7 +86,7 @@ The core counting logic is available as a library. It can be imported into other Go projects. ```go -import "github.com/moderrek/lines/pkg/lines" +import "github.com/Moderrek/lines/pkg/lines" ``` ### Example @@ -98,7 +98,7 @@ import ( "fmt" "log" - "github.com/moderrek/lines/pkg/lines" + "github.com/Moderrek/lines/pkg/lines" ) func main() { diff --git a/cmd/lines/cli.go b/cmd/lines/cli.go index 6eca4ff..ace7325 100644 --- a/cmd/lines/cli.go +++ b/cmd/lines/cli.go @@ -20,7 +20,7 @@ type cliOptions struct { func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, error) { opts := &cliOptions{} - fs := flag.NewFlagSet(PROGRAM_NAME, flag.ContinueOnError) + fs := flag.NewFlagSet(ProgramName, flag.ContinueOnError) fs.SetOutput(stderr) fs.StringVar(&opts.dir, "dir", ".", "The directory to analyze") diff --git a/cmd/lines/config.go b/cmd/lines/config.go index af3d592..4c52054 100644 --- a/cmd/lines/config.go +++ b/cmd/lines/config.go @@ -1,5 +1,5 @@ package main -const PROGRAM_NAME = "lines" -const VERSION = "dev-v1.3.0" -const AUTHOR = "Tymon Wozniak @Moderrek" +const ProgramName = "lines" +const Version = "dev-v1.3.0" +const Author = "Tymon Wozniak @Moderrek" diff --git a/cmd/lines/output.go b/cmd/lines/output.go index 18c3fb9..c7d1272 100644 --- a/cmd/lines/output.go +++ b/cmd/lines/output.go @@ -6,8 +6,8 @@ import ( "io" "sort" + "github.com/Moderrek/lines/pkg/lines" "github.com/fatih/color" - "github.com/moderrek/lines/pkg/lines" ) func printJSONOutput(w io.Writer, result *lines.Result) error { diff --git a/cmd/lines/run.go b/cmd/lines/run.go index 40ffa4b..188f8ef 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -6,9 +6,9 @@ import ( "os" "time" + "github.com/Moderrek/lines/pkg/lines" "github.com/fatih/color" "github.com/mattn/go-isatty" - "github.com/moderrek/lines/pkg/lines" ) func run(stdout, stderr io.Writer, args []string) error { @@ -17,6 +17,11 @@ func run(stdout, stderr io.Writer, args []string) error { return err } + targets := fs.Args() + if len(targets) == 0 { + targets = []string{"."} + } + isStdoutTerminal := isatty.IsTerminal(os.Stdout.Fd()) isStderrTerminal := isatty.IsTerminal(os.Stderr.Fd()) @@ -24,12 +29,12 @@ func run(stdout, stderr io.Writer, args []string) error { color.NoColor = !useColor if opts.version { - fmt.Printf("%s version %s created by %s\n", PROGRAM_NAME, VERSION, AUTHOR) + fmt.Printf("%s version %s created by %s\n", ProgramName, Version, Author) return nil } if opts.help { - fmt.Fprintf(stdout, "Usage: %s [options]\n", PROGRAM_NAME) + fmt.Fprintf(stdout, "Usage: %s [options]\n", ProgramName) fs.SetOutput(stdout) fs.PrintDefaults() return nil @@ -52,7 +57,7 @@ func run(stdout, stderr io.Writer, args []string) error { go startProgressReporter(stderr, stopProgress, startTime, counter) } - result, err := counter.Run(opts.dir) + result, err := counter.Run(targets) if err != nil { return err } diff --git a/go.mod b/go.mod index 28b2e79..7cfad30 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/moderrek/lines +module github.com/Moderrek/lines go 1.22.4 diff --git a/pkg/lines/config.go b/pkg/lines/config.go index ab5ddc6..b927f4b 100644 --- a/pkg/lines/config.go +++ b/pkg/lines/config.go @@ -1,5 +1,9 @@ package lines +// TODO: consider storing ignored directories and extensions as slices instead of maps. +// We can convert them to maps during initialization. +// This would make the configuration more user-friendly while still allowing for efficient lookups during analysis. + // Config holds settings for the line counting process. type Config struct { // IncludeHidden analyzes hidden files and directories (starting with '.'). diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go index 47844c7..510694e 100644 --- a/pkg/lines/counter.go +++ b/pkg/lines/counter.go @@ -1,6 +1,7 @@ package lines import ( + "errors" "fmt" "os" "path/filepath" @@ -9,6 +10,8 @@ import ( "sync/atomic" ) +// TODO: add handle log function. + // Counter analyzes directories and counts non-blank lines of code. type Counter struct { Config Config @@ -21,6 +24,8 @@ type Counter struct { FilesFound atomic.Int64 FilesProcessed atomic.Int64 + + isWorking bool } // NewCounter creates a new Counter with the given configuration. @@ -53,35 +58,87 @@ func configWithDefaults(config Config) Config { return config } -// Run analyzes the given directory and returns the results. -// It recursively walks the directory tree using goroutines for performance. -func (c *Counter) Run(dir string) (*Result, error) { - c.reset() +// checkTargets checks does target exists and it is directory or file. +func checkTargets(targets []string) (map[string]bool, error) { + isDir := make(map[string]bool) + for _, target := range targets { + fileInfo, err := os.Stat(target) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("target does not exists %q: %v\n", filepath.ToSlash(target), err) + } + return nil, err + } + isDir[target] = fileInfo.IsDir() + } + return isDir, nil +} - if _, err := os.Stat(dir); os.IsNotExist(err) { - return nil, fmt.Errorf("directory %q does not exist", dir) +func (c *Counter) Run(targets []string) (*Result, error) { + isDir, err := checkTargets(targets) + if err != nil { + return nil, err } - numWorkers := c.Config.NumWorkers - maximumWaitingWork := numWorkers * 4 - c.filesToAnalyze = make(chan string, maximumWaitingWork) + // initializes worker pool + c.reset() - c.logVerbosef("creating %d workers with work queue of capacity %d", numWorkers, maximumWaitingWork) - for i := 0; i < numWorkers; i++ { + workersCount := c.Config.NumWorkers + maximumWaitingWork := workersCount * 4 + c.filesToAnalyze = make(chan string, maximumWaitingWork) + c.logVerbosef("creating %d workers with work queue of capacity %d", workersCount, maximumWaitingWork) + for range workersCount { c.workers.Add(1) go c.analyzeFilesWorker() } + combinedResult := Result{} + for _, target := range targets { + if isDir[target] { + err := c.walkDir(target) + if err != nil { + return nil, err + } + c.workers.Wait() + + result := Result{ + c.lines, + } + + combinedResult = MergeResults(combinedResult, result) + } else { + c.FilesFound.Add(1) + c.filesToAnalyze <- target + + } + + } + + close(c.filesToAnalyze) + c.workers.Wait() + + return &combinedResult, nil +} + +func (c *Counter) startAnalysis() { + c.reset() +} + +// analyzeDirectory analyzes the given directory and returns the results. +// It recursively walks the directory tree using goroutines for performance. +func (c *Counter) analyzeDirectory(dir string) (*Result, error) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return nil, fmt.Errorf("directory %q does not exist", dir) + } + c.logVerbosef("starting analysis at: %q", filepath.ToSlash(dir)) err := c.walkDir(dir) if err != nil { return nil, err } - close(c.filesToAnalyze) - c.workers.Wait() // Makes copy of result. - c.logVerbosef("coping result") + c.logVerbosef("copying result") linesByExt := make(map[string]int) for ext, count := range c.lines { linesByExt[ext] = count diff --git a/pkg/lines/log.go b/pkg/lines/log.go index 3ac0576..00a1d17 100644 --- a/pkg/lines/log.go +++ b/pkg/lines/log.go @@ -5,8 +5,12 @@ import ( "os" ) +// TODO: use writer from config. + +// logVerbosef logs a formatted message if verbose mode is enabled in config. func (c *Counter) logVerbosef(format string, v ...any) { - if c.Config.Verbose { - fmt.Fprintf(os.Stderr, "info: "+format+"\n", v...) + if !c.Config.Verbose { + return } + fmt.Fprintf(os.Stderr, "info: "+format+"\n", v...) } diff --git a/pkg/lines/result.go b/pkg/lines/result.go index 70e7f4f..8d95302 100644 --- a/pkg/lines/result.go +++ b/pkg/lines/result.go @@ -1,7 +1,23 @@ package lines +// TODO: add processed files count. +// TODO: add total lines count. + // Result represents the results of the line counting process. type Result struct { // LinesByExtension maps file extensions to their total line counts. LinesByExtension map[string]int } + +// MergeResults merges multiple results into single new result. +func MergeResults(results ...Result) Result { + merged := Result{ + LinesByExtension: make(map[string]int), + } + for _, result := range results { + for ext, count := range result.LinesByExtension { + merged.LinesByExtension[ext] += count + } + } + return merged +} diff --git a/pkg/lines/walker.go b/pkg/lines/walker.go index 9ce3778..5f4ee84 100644 --- a/pkg/lines/walker.go +++ b/pkg/lines/walker.go @@ -29,19 +29,24 @@ func (c *Counter) walkDir(dir string) error { c.logVerbosef("skipping ignored directory: %q", filepath.ToSlash(path)) return filepath.SkipDir } + + // continue walking return nil } if d.Type().IsRegular() && c.shouldAnalyzeFile(path, d.Name()) { + // Enqueue the file for analysis. c.logVerbosef("found file to analyze: %q", filepath.ToSlash(path)) c.FilesFound.Add(1) c.filesToAnalyze <- path } else { + // File is skipped because it is not a regular file or its extension is ignored. c.logVerbosef("skipping file: %q", filepath.ToSlash(path)) } return nil }) + return err } From ee5aebdf96a537a934ee3e6bc3ae6dbdae042bab Mon Sep 17 00:00:00 2001 From: Tymon Wozniak Date: Wed, 29 Jul 2026 19:45:16 +0200 Subject: [PATCH 17/17] feat: enhance command-line options and update README for v1.3.0 release --- .github/workflows/go.yml | 2 ++ README.md | 60 ++++++++++++++++++++++++++++------------ cmd/lines/cli.go | 16 +++++++++-- cmd/lines/config.go | 2 +- cmd/lines/run.go | 12 +++++--- pkg/lines/counter.go | 24 +++++++++------- pkg/lines/defaults.go | 31 ++++++++++++++++++--- 7 files changed, 109 insertions(+), 38 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 25901b1..da6d7b6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -38,6 +38,8 @@ jobs: needs: build runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write steps: - uses: actions/checkout@v4 with: diff --git a/README.md b/README.md index 2737478..a8d0ed5 100644 --- a/README.md +++ b/README.md @@ -22,18 +22,34 @@ The tool is designed for performance, utilizing goroutines to process files in p To install the `lines` command-line tool, ensure you have [Go](https://go.dev/doc/install) installed and configured, then run: ```shell -go install github.com/Moderrek/lines/cmd/lines@latest +go install github.com/Moderrek/lines/cmd/lines@v1.3.0 ``` -This will download the source, compile it, and place the `lines` binary in your Go bin directory (`$GOPATH/bin` or `$HOME/go/in`). +This will download the source, compile it, and place the `lines` binary in your Go bin directory (`$GOPATH/bin` or `$HOME/go/bin`). The binary works on Linux and Windows, and the release assets are published for both platforms. + +If you want to use the library in another Go module, add it with: + +```shell +go get github.com/Moderrek/lines@v1.3.0 +``` + +Then import the package from `pkg/lines`: + +```go +import "github.com/Moderrek/lines/pkg/lines" +``` ## Usage -The `lines` command accepts the following flags: +The `lines` command accepts the following flags and targets: +```text +Usage: lines [options] [--dir PATH ...] [PATH ...] ``` -Usage: lines [options] +You can analyze multiple directories or files in one run by repeating `--dir` or passing positional arguments. + +``` Options: -color Force color output (e.g. when piping) @@ -55,6 +71,10 @@ Options: Print the version ``` +```shell +lines --dir ./cmd --dir ./pkg ./README.md +``` + ### Example To analyze the directory `~/projects/my-app` and display the top 5 extensions: @@ -85,10 +105,6 @@ Example output (`--json`): The core counting logic is available as a library. It can be imported into other Go projects. -```go -import "github.com/Moderrek/lines/pkg/lines" -``` - ### Example ```go @@ -128,20 +144,14 @@ You can customize which directories and file extensions to ignore: ```go config := lines.Config{ IncludeHidden: false, - IgnoredDirs: map[string]struct{}{ - "node_modules": {}, - ".git": {}, - }, - IgnoredExtensions: map[string]struct{}{ - ".exe": {}, - ".env": {}, - }, + IgnoredDirs: lines.IgnoredDirSet("node_modules", ".git"), + IgnoredExtensions: lines.IgnoredExtensionSet("exe", ".env"), } counter := lines.NewCounter(config) result, err := counter.Run("./src") ``` -If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensible defaults. +If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensible defaults. The helper functions above are optional, but they make the config easier to read. ## Building from Source @@ -159,6 +169,22 @@ If `IgnoredDirs` or `IgnoredExtensions` are not provided, the library uses sensi ``` This will create a `lines` executable in the current directory. +### Releasing + +For v1.3.0, the intended distribution flow is: + +```shell +go install github.com/Moderrek/lines/cmd/lines@v1.3.0 +``` + +Or, if you are integrating the library into another module: + +```shell +go get github.com/Moderrek/lines@v1.3.0 +``` + +The GitHub release pipeline builds binaries for Linux and Windows so users without Go installed can download a ready-made executable. + ## License This project is licensed under the MIT License. diff --git a/cmd/lines/cli.go b/cmd/lines/cli.go index ace7325..5fd5481 100644 --- a/cmd/lines/cli.go +++ b/cmd/lines/cli.go @@ -3,10 +3,11 @@ package main import ( "flag" "io" + "strings" ) type cliOptions struct { - dir string + dirs multiStringFlag version bool help bool hidden bool @@ -18,12 +19,23 @@ type cliOptions struct { verbose bool } +type multiStringFlag []string + +func (m *multiStringFlag) String() string { + return strings.Join(*m, ",") +} + +func (m *multiStringFlag) Set(value string) error { + *m = append(*m, value) + return nil +} + func parseFlags(stderr io.Writer, args []string) (*cliOptions, *flag.FlagSet, error) { opts := &cliOptions{} fs := flag.NewFlagSet(ProgramName, flag.ContinueOnError) fs.SetOutput(stderr) - fs.StringVar(&opts.dir, "dir", ".", "The directory to analyze") + fs.Var(&opts.dirs, "dir", "The directories or files to analyze. Can be repeated.") fs.BoolVar(&opts.version, "version", false, "Print the version and exit") fs.BoolVar(&opts.version, "v", false, "Print the version and exit") fs.BoolVar(&opts.help, "help", false, "Print the help message and exit") diff --git a/cmd/lines/config.go b/cmd/lines/config.go index 4c52054..cec46d0 100644 --- a/cmd/lines/config.go +++ b/cmd/lines/config.go @@ -1,5 +1,5 @@ package main const ProgramName = "lines" -const Version = "dev-v1.3.0" +const Version = "v1.3.0" const Author = "Tymon Wozniak @Moderrek" diff --git a/cmd/lines/run.go b/cmd/lines/run.go index 188f8ef..4ec1ab1 100644 --- a/cmd/lines/run.go +++ b/cmd/lines/run.go @@ -17,7 +17,9 @@ func run(stdout, stderr io.Writer, args []string) error { return err } - targets := fs.Args() + targets := make([]string, 0, len(opts.dirs)+len(fs.Args())) + targets = append(targets, opts.dirs...) + targets = append(targets, fs.Args()...) if len(targets) == 0 { targets = []string{"."} } @@ -34,7 +36,10 @@ func run(stdout, stderr io.Writer, args []string) error { } if opts.help { - fmt.Fprintf(stdout, "Usage: %s [options]\n", ProgramName) + fmt.Fprintf(stdout, "Usage: %s [options] [--dir PATH ...] [PATH ...]\n", ProgramName) + fmt.Fprintln(stdout, "") + fmt.Fprintln(stdout, "You can pass multiple directories or files using repeated --dir flags or positional arguments.") + fmt.Fprintln(stdout, "") fs.SetOutput(stdout) fs.PrintDefaults() return nil @@ -48,7 +53,6 @@ func run(stdout, stderr io.Writer, args []string) error { counter := lines.NewCounter(config) stopProgress := make(chan struct{}) - defer close(stopProgress) startTime := time.Now() @@ -63,7 +67,7 @@ func run(stdout, stderr io.Writer, args []string) error { } if showProgress { - stopProgress <- struct{}{} + close(stopProgress) reportProgress(stderr, startTime, counter) fmt.Fprintf(stderr, "\n") } diff --git a/pkg/lines/counter.go b/pkg/lines/counter.go index 510694e..1019cf8 100644 --- a/pkg/lines/counter.go +++ b/pkg/lines/counter.go @@ -75,6 +75,10 @@ func checkTargets(targets []string) (map[string]bool, error) { } func (c *Counter) Run(targets []string) (*Result, error) { + if len(targets) == 0 { + targets = []string{"."} + } + isDir, err := checkTargets(targets) if err != nil { return nil, err @@ -92,24 +96,17 @@ func (c *Counter) Run(targets []string) (*Result, error) { go c.analyzeFilesWorker() } - combinedResult := Result{} for _, target := range targets { if isDir[target] { err := c.walkDir(target) if err != nil { + close(c.filesToAnalyze) + c.workers.Wait() return nil, err } - c.workers.Wait() - - result := Result{ - c.lines, - } - - combinedResult = MergeResults(combinedResult, result) } else { c.FilesFound.Add(1) c.filesToAnalyze <- target - } } @@ -117,7 +114,14 @@ func (c *Counter) Run(targets []string) (*Result, error) { close(c.filesToAnalyze) c.workers.Wait() - return &combinedResult, nil + result := Result{LinesByExtension: make(map[string]int)} + c.linesLock.Lock() + for ext, count := range c.lines { + result.LinesByExtension[ext] = count + } + c.linesLock.Unlock() + + return &result, nil } func (c *Counter) startAnalysis() { diff --git a/pkg/lines/defaults.go b/pkg/lines/defaults.go index 82ce3c1..616f1f4 100644 --- a/pkg/lines/defaults.go +++ b/pkg/lines/defaults.go @@ -2,7 +2,7 @@ package lines import "strings" -// DefaultIgnoredDirs returns slice of default directories to ignore. +// DefaultIgnoredDirs returns the default directories to ignore. func DefaultIgnoredDirs() map[string]struct{} { return map[string]struct{}{ "node_modules": {}, @@ -12,7 +12,7 @@ func DefaultIgnoredDirs() map[string]struct{} { } } -// DefaultIgnoredExtensions returns set of default file extensions to ignore. +// DefaultIgnoredExtensions returns the default file extensions to ignore. func DefaultIgnoredExtensions() map[string]struct{} { return makeExtensionSet( "exe", "dll", "so", "dylib", "msi", "mui", "mun", @@ -39,13 +39,36 @@ func DefaultIgnoredExtensions() map[string]struct{} { ) } -// makeExtensionSet makes set of file extensions for faster search. +// IgnoredDirSet builds a directory ignore set from a list of directory names. +func IgnoredDirSet(items ...string) map[string]struct{} { + return makeStringSet(items...) +} + +// IgnoredExtensionSet builds a file extension ignore set from a list of extensions. +// Extensions may be passed with or without a leading dot. +func IgnoredExtensionSet(items ...string) map[string]struct{} { + return makeExtensionSet(items...) +} + +// makeExtensionSet makes a set of file extensions for faster lookup. // The extensions are stored in lowercase and prefixed with a dot. // NOTE: Extensions are prefixed with dot to avoid stripping out the dot for every file. func makeExtensionSet(items ...string) map[string]struct{} { set := make(map[string]struct{}, len(items)) for _, item := range items { - set["."+strings.ToLower(item)] = struct{}{} + normalized := strings.TrimPrefix(strings.ToLower(item), ".") + if normalized == "" { + continue + } + set["."+normalized] = struct{}{} + } + return set +} + +func makeStringSet(items ...string) map[string]struct{} { + set := make(map[string]struct{}, len(items)) + for _, item := range items { + set[item] = struct{}{} } return set }