diff --git a/Modelfile.pro b/Modelfile.pro index 987a3be..d55287d 100644 --- a/Modelfile.pro +++ b/Modelfile.pro @@ -35,53 +35,49 @@ As an agent with tools: read the relevant files before editing them, change as l Format: each file goes in one fenced code block with a language tag, preceded by its path. Keep explanations short and write them in the user's language.""" # One worked exchange that shows the expected shape of an answer: contract, code, tests, how to run them. -MESSAGE user """Write a Python function parse_duration(text) that turns durations like "1h30m" or "45s" into seconds. Invalid input must raise ValueError.""" -MESSAGE assistant """Assumption: units are h, m and s, each used at most once and in that order, with no spaces between parts ("1h 30m" is rejected); a non-string raises TypeError. +MESSAGE user """Write a Python function is_valid_ipv4(text) that returns True for a valid dotted-quad IPv4 address and False otherwise.""" +MESSAGE assistant """Assumption: four decimal parts from 0 to 255 separated by dots, ASCII digits only, no leading zeros ("01.2.3.4" is rejected because some tools read it as octal), no surrounding spaces; a non-string raises TypeError. -`duration.py` +`ipv4.py` ```python -import re - -_SECONDS = {"h": 3600, "m": 60, "s": 1} -_PART = re.compile(r"(\d+)([hms])") - - -def parse_duration(text: str) -> int: - '''Return the number of seconds in a duration such as "1h30m" or "45s".''' +def is_valid_ipv4(text: str) -> bool: + '''Return True if text is a dotted-quad IPv4 address such as "192.168.0.1".''' if not isinstance(text, str): raise TypeError("text must be a string") - value = text.strip().lower() - parts = _PART.findall(value) - units = [unit for _, unit in parts] - # Every character must belong to a part, and units must appear once, in h-m-s order. - if not parts or _PART.sub("", value) or units != sorted(set(units), key="hms".index): - raise ValueError(f"invalid duration: {text!r}") - return sum(int(number) * _SECONDS[unit] for number, unit in parts) + parts = text.split(".") + if len(parts) != 4: + return False + for part in parts: + # ASCII digits only (str.isdigit also accepts other scripts), no sign, no leading zero. + if not part.isascii() or not part.isdigit() or (len(part) > 1 and part[0] == "0"): + return False + if int(part) > 255: + return False + return True ``` -`test_duration.py` +`test_ipv4.py` ```python import pytest -from duration import parse_duration +from ipv4 import is_valid_ipv4 -@pytest.mark.parametrize("text, seconds", [ - ("45s", 45), ("1h30m", 5400), ("2h", 7200), (" 10M ", 600), ("1h0m5s", 3605), ("0s", 0), -]) -def test_valid(text, seconds): - assert parse_duration(text) == seconds +@pytest.mark.parametrize("text", ["0.0.0.0", "192.168.0.1", "255.255.255.255", "10.0.0.10", "1.2.3.4"]) +def test_valid(text): + assert is_valid_ipv4(text) -@pytest.mark.parametrize("text", ["", " ", "abc", "10", "1h1h", "30m1h", "1.5h", "-5s", "1h 30m"]) +@pytest.mark.parametrize("text", [ + "", "1.2.3", "1.2.3.4.5", "256.1.1.1", "01.2.3.4", "1.2.3.-4", "1.2.3.4 ", "a.b.c.d", "1..2.3", "١.2.3.4", +]) def test_invalid(text): - with pytest.raises(ValueError): - parse_duration(text) + assert not is_valid_ipv4(text) def test_not_a_string(): with pytest.raises(TypeError): - parse_duration(90) + is_valid_ipv4(1234) ``` -Run: `python -m pytest test_duration.py`. Limit: no days or fractional values; add them to `_SECONDS` and the pattern if you need them.""" +Run: `python -m pytest test_ipv4.py`. Limit: IPv6 and CIDR suffixes such as "/24" are rejected; use the standard `ipaddress` module if you need them.""" diff --git a/docs/bench/cpu-nothink-0.3.json b/docs/bench/cpu-nothink-0.3.json index 0d1fbce..11a97d9 100644 --- a/docs/bench/cpu-nothink-0.3.json +++ b/docs/bench/cpu-nothink-0.3.json @@ -11,14 +11,14 @@ { "summary": { "model": "flycoder0.3", - "passed": 1, - "total": 3, - "passRate": 0.3333333333333333, - "decodeTokensPerSecond": 3.7353407093832978, - "prefillTokensPerSecond": 21.755343865522967, - "meanOutputTokens": 588, - "meanSecondsPerProblem": 180.223, - "truncated": 0 + "passed": 7, + "total": 12, + "passRate": 0.5833333333333334, + "decodeTokensPerSecond": 3.844831517257886, + "prefillTokensPerSecond": 22.80421580322576, + "meanOutputTokens": 745.9166666666666, + "meanSecondsPerProblem": 215.52991666666665, + "truncated": 1 }, "results": [ { @@ -65,6 +65,141 @@ "prefillMs": 23983.098, "thinkingChars": 0, "answer": "```python\nimport re\n\ndef parse_duration(text: str) -> int:\n \"\"\"\n Parse a duration string into total seconds.\n \n Supports units: d (days), h (hours), m (minutes), s (seconds).\n Units must appear in order (d, h, m, s) and at most once each.\n Spaces are allowed between parts or around the string.\n \n Raises ValueError for invalid input.\n \"\"\"\n # Strip leading/trailing whitespace\n text = text.strip()\n \n if not text:\n raise ValueError(\"Empty string is invalid\")\n \n # Split by spaces to handle \"1h 15m 10s\"\n parts = text.split()\n \n if not parts:\n raise ValueError(\"Invalid input after stripping\")\n \n total_seconds = 0\n seen_units = set()\n expected_order = ['d', 'h', 'm', 's']\n \n # Regex to match number followed by a unit\n pattern = re.compile(r'^(\\d+)([dhms])$')\n \n for part in parts:\n match = pattern.match(part)\n if not match:\n raise ValueError(f\"Invalid part: '{part}'\")\n \n num = int(match.group(1))\n unit = match.group(2)\n \n if num < 0:\n raise ValueError(f\"Negative number not allowed in part: {part}\")\n \n # Check unit validity\n if unit not in expected_order:\n raise ValueError(f\"Invalid unit: '{unit}'\")\n \n if unit in seen_units:\n raise ValueError(f\"Duplicate unit: '{unit}'\")\n \n seen_units.add(unit)\n \n # Validate order: if a unit is seen, next valid unit must follow in order\n # We track the last seen unit index to ensure order\n unit_index = expected_order.index(unit)\n for prev_unit in seen_units:\n if expected_order.index(prev_unit) > unit_index:\n raise ValueError(f\"Units out of order: '{unit}' appears after '{prev_unit}'\")\n \n # Add to total\n unit_to_seconds = {'d': 86400, 'h': 3600, 'm': 60, 's': 1}\n total_seconds += num * unit_to_seconds[unit]\n \n # Final validation: ensure all expected units were not required but some were missing?\n # The problem says \"one or more ... parts\", so missing units are OK as long as order is respected.\n # However, we must ensure no extra characters remain.\n \n if len(seen_units) == 0:\n raise ValueError(\"No valid unit found in input\")\n \n return total_seconds\n```" + }, + { + "problem": "py-topo-sort", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 315601, + "loadMs": 10, + "outputTokens": 1111, + "decodeMs": 294001.411, + "promptTokens": 488, + "prefillMs": 21462.385, + "thinkingChars": 0, + "answer": "```python\ndef topo_sort(deps):\n \"\"\"\n Perform topological sort on a directed graph defined by dependencies.\n \n Args:\n deps (dict): Maps each node (str) to a list of nodes it depends on.\n Nodes that only appear as dependencies are also included.\n \n Returns:\n list: A list of nodes in topological order (alphabetically smallest \n among ready nodes first).\n \n Raises:\n ValueError: If the graph contains a cycle.\n \"\"\"\n # Build the complete set of nodes and the adjacency list (for Kahn's algorithm)\n # deps[u] = [v1, v2, ...] means u depends on v1, v2, ...\n # In graph terms: v -> u (edge from dependency to dependent)\n \n # Collect all nodes\n all_nodes = set(deps.keys())\n for dependencies in deps.values():\n for node in dependencies:\n all_nodes.add(node)\n \n # Build reverse adjacency list: for each node, who depends on it?\n # And compute in-degrees based on the dependency edges (dependency -> dependent)\n # If A depends on B, there is an edge B -> A.\n \n adj = {node: [] for node in all_nodes}\n in_degree = {node: 0 for node in all_nodes}\n \n for node, dependencies in deps.items():\n for dep in dependencies:\n # dep -> node\n adj[dep].append(node)\n in_degree[node] += 1\n \n # Initialize queue with nodes having in-degree 0, sorted alphabetically\n queue = sorted([node for node in all_nodes if in_degree[node] == 0])\n \n result = []\n \n while queue:\n # Pop the alphabetically smallest node\n current = queue.pop(0)\n result.append(current)\n \n # Reduce in-degrees of neighbors\n for neighbor in adj[current]:\n in_degree[neighbor] -= 1\n if in_degree[neighbor] == 0:\n # Insert in sorted order to maintain alphabetical property\n # Since we pop from the front, we can just append and re-sort or use bisect\n # Given the requirement for deterministic alphabetical order, we'll re-sort \n # or insert in place. For simplicity and correctness with potentially many nodes,\n # we append and then sort the queue, or use a heap. \n # However, since we need the smallest first, and the queue size might grow,\n # let's append and then sort the queue again. \n # Alternatively, use a min-heap for O(log n) pops.\n # Given the requirement for simplicity and correctness, let's use a sorted list \n # and re-insert while maintaining order.\n # But re-sorting the entire queue every time is O(N^2) in worst case.\n # Let's use a min-heap approach for efficiency.\n pass\n \n # Re-queue the newly available nodes in sorted order\n # To maintain O(N log N) overall, we can use a heap.\n # But since we need to pop the smallest, let's manage the queue carefully.\n # Actually, let's restart the loop logic with a heap for efficiency.\n # However, the problem asks for a simple function. Let's stick to the sorted list \n # but insert new nodes in sorted order.\n # We'll collect new zero-degree nodes and insert them in sorted order.\n \n # Wait, the above logic inside the loop is flawed because I didn't update the queue with new zero-degree nodes.\n # Let's rewrite the loop logic properly.\n pass\n \n # Let's restart the implementation with a correct approach using a sorted list or heap.\n # Since the requirement is to pick the alphabetically smallest, a min-heap is ideal.\n \n import heapq\n \n # Reset for correct implementation\n all_nodes = set(deps.keys())\n for dependencies in deps.values():\n for node in dependencies:\n all_nodes.add(node)\n \n adj = {node: [] for node in all_nodes}\n in_degree = {node: 0 for node in all_nodes}\n \n for node, dependencies in deps.items():\n for dep in dependencies:\n adj[dep].append(node)\n in_degree[node] += 1\n \n # Use a min-heap for the zero-in-degree nodes\n heap = []\n for node in all_nodes:\n if in_degree[node] == 0:\n heapq.heappush(heap, node)\n \n result = []\n \n while heap:\n current = heapq.heappop(heap)\n result.append(current)\n \n for neighbor in adj[current]:\n in_degree[neighbor] -= 1\n if in_degree[neighbor] == 0:\n heapq.heappush(heap, neighbor)\n \n # Check for cycle\n if len(result) != len(all_nodes):\n raise ValueError(\"Cycle detected in the dependency graph.\")\n \n return result\n```" + }, + { + "problem": "py-lru-cache", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 64053, + "loadMs": 5, + "outputTokens": 176, + "decodeMs": 43828.919, + "promptTokens": 493, + "prefillMs": 20067.182, + "thinkingChars": 0, + "answer": "```python\nfrom collections import OrderedDict\n\n\nclass LRUCache:\n def __init__(self, capacity: int):\n if capacity < 0:\n raise ValueError(\"Capacity must be non-negative.\")\n self.capacity = capacity\n self.cache = OrderedDict()\n\n def get(self, key: int) -> int:\n if key in self.cache:\n self.cache.move_to_end(key)\n return self.cache[key]\n return -1\n\n def put(self, key: int, value: int) -> None:\n if key in self.cache:\n self.cache.move_to_end(key)\n self.cache[key] = value\n else:\n self.cache[key] = value\n if len(self.cache) > self.capacity:\n self.cache.popitem(last=False)\n```" + }, + { + "problem": "py-wrap-text", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 158502, + "loadMs": 5, + "outputTokens": 536, + "decodeMs": 137462.126, + "promptTokens": 501, + "prefillMs": 21021.449, + "thinkingChars": 0, + "answer": "```python\ndef wrap_text(text, width):\n \"\"\"\n Wraps text greedily into lines.\n \n - Words are separated by any whitespace; runs of whitespace collapse.\n - Lines never start or end with a space.\n - As many words as fit on each line (joined by single spaces, line length <= width).\n - A word longer than width is split into pieces of exactly 'width' characters \n (last piece may be shorter); each piece is treated as a word.\n - Returns a list of lines (strings).\n - Raises ValueError if width < 1.\n \n Args:\n text (str): The input text to wrap.\n width (int): The maximum line width.\n \n Returns:\n List[str]: A list of wrapped lines.\n \n Raises:\n ValueError: If width is less than 1.\n \"\"\"\n if width < 1:\n raise ValueError(\"width must be at least 1\")\n\n if not text or text.isspace():\n return []\n\n # Split text into words (handling any whitespace separator)\n words = text.split()\n \n # Process long words: split them into chunks of 'width'\n processed_words = []\n for word in words:\n if len(word) <= width:\n processed_words.append(word)\n else:\n # Split into chunks of exactly 'width', last chunk can be shorter\n for i in range(0, len(word), width):\n processed_words.append(word[i:i + width])\n\n if not processed_words:\n return []\n\n lines = []\n current_line = []\n current_length = 0\n\n for word in processed_words:\n word_len = len(word)\n \n # Calculate new length if we add this word (including space if not first)\n if current_line:\n new_length = current_length + 1 + word_len\n else:\n new_length = word_len\n \n if new_length <= width:\n current_line.append(word)\n current_length = new_length\n else:\n # Finalize current line\n lines.append(' '.join(current_line))\n # Start new line with current word\n current_line = [word]\n current_length = word_len\n\n # Add the last line if there are any words left\n if current_line:\n lines.append(' '.join(current_line))\n\n return lines\n```" + }, + { + "problem": "py-semver-compare", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 263044, + "loadMs": 5, + "outputTokens": 949, + "decodeMs": 240905.057, + "promptTokens": 510, + "prefillMs": 21986.479, + "thinkingChars": 0, + "answer": "```python\ndef compare_versions(a: str, b: str) -> int:\n \"\"\"\n Compare two Semantic Versioning 2.0.0 strings.\n \n Returns:\n -1 if a < b\n 0 if a == b\n 1 if a > b\n \n SemVer precedence rules:\n 1. Core (MAJOR.MINOR.PATCH) is compared numerically.\n 2. A pre-release version has lower precedence than a normal version.\n 3. Pre-release identifiers are compared dot by dot:\n - Numeric identifiers are compared as integers.\n - Alphanumeric identifiers are compared as ASCII strings.\n - Numeric identifiers are lower precedence than alphanumeric ones.\n 4. A shorter set of pre-release identifiers is lower precedence if all preceding ones are equal.\n 5. Build metadata is ignored.\n \"\"\"\n \n def parse_version(version_str: str):\n # Remove build metadata (everything after +)\n if '+' in version_str:\n version_str = version_str.split('+')[0]\n \n # Split into core and pre-release\n parts = version_str.split('-')\n core = parts[0]\n prerelease = parts[1] if len(parts) > 1 else None\n \n # Parse core (MAJOR.MINOR.PATCH)\n core_parts = core.split('.')\n major = int(core_parts[0])\n minor = int(core_parts[1])\n patch = int(core_parts[2])\n \n # Parse pre-release identifiers\n prerelease_identifiers = None\n if prerelease is not None:\n identifiers = []\n for ident in prerelease.split('.'):\n if ident.isdigit():\n identifiers.append(('numeric', int(ident)))\n else:\n identifiers.append(('alphanumeric', ident))\n prerelease_identifiers = identifiers\n \n return {\n 'major': major,\n 'minor': minor,\n 'patch': patch,\n 'prerelease': prerelease_identifiers\n }\n \n parsed_a = parse_version(a)\n parsed_b = parse_version(b)\n \n # Compare core version\n if parsed_a['major'] != parsed_b['major']:\n return -1 if parsed_a['major'] < parsed_b['major'] else 1\n if parsed_a['minor'] != parsed_b['minor']:\n return -1 if parsed_a['minor'] < parsed_b['minor'] else 1\n if parsed_a['patch'] != parsed_b['patch']:\n return -1 if parsed_a['patch'] < parsed_b['patch'] else 1\n \n # Core versions are equal, compare pre-release\n # No pre-release has higher precedence than a pre-release\n if parsed_a['prerelease'] is None and parsed_b['prerelease'] is not None:\n return 1\n if parsed_a['prerelease'] is not None and parsed_b['prerelease'] is None:\n return -1\n if parsed_a['prerelease'] is None and parsed_b['prerelease'] is None:\n return 0\n \n # Both have pre-release, compare identifiers\n ids_a = parsed_a['prerelease']\n ids_b = parsed_b['prerelease']\n \n min_len = min(len(ids_a), len(ids_b))\n \n for i in range(min_len):\n type_a, val_a = ids_a[i]\n type_b, val_b = ids_b[i]\n \n # Numeric identifiers have lower precedence than alphanumeric\n if type_a != type_b:\n if type_a == 'numeric':\n return -1\n else:\n return 1\n \n # Same type, compare values\n if type_a == 'numeric':\n if val_a != val_b:\n return -1 if val_a < val_b else 1\n else:\n # Alphanumeric: ASCII order\n if val_a != val_b:\n return -1 if val_a < val_b else 1\n \n # All compared identifiers are equal\n # Shorter set of identifiers is lower precedence\n if len(ids_a) < len(ids_b):\n return -1\n elif len(ids_a) > len(ids_b):\n return 1\n else:\n return 0\n```" + }, + { + "problem": "py-spiral", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 104729, + "loadMs": 5, + "outputTokens": 334, + "decodeMs": 85473.382, + "promptTokens": 428, + "prefillMs": 19080.333, + "thinkingChars": 0, + "answer": "```python\ndef spiral_order(matrix):\n \"\"\"\n Returns the elements of a rectangular matrix in clockwise spiral order.\n \n Args:\n matrix (List[List[int]]): A list of lists representing a rectangular matrix.\n \n Returns:\n List: Elements in spiral order. Returns [] for empty matrix or matrix of empty rows.\n \"\"\"\n if not matrix or not matrix[0]:\n return []\n\n top, bottom = 0, len(matrix) - 1\n left, right = 0, len(matrix[0]) - 1\n result = []\n\n while top <= bottom and left <= right:\n # Traverse from left to right along the top row\n for col in range(left, right + 1):\n result.append(matrix[top][col])\n top += 1\n\n # Traverse from top to bottom along the right column\n if top <= bottom:\n for row in range(top, bottom + 1):\n result.append(matrix[row][right])\n right -= 1\n\n # Traverse from right to left along the bottom row\n if top <= bottom:\n for col in range(right, left - 1, -1):\n result.append(matrix[bottom][col])\n bottom -= 1\n\n # Traverse from bottom to top along the left column\n if left <= right:\n for row in range(bottom, top - 1, -1):\n result.append(matrix[row][left])\n left += 1\n\n return result\n```" + }, + { + "problem": "py-csv-line", + "sample": 0, + "ok": false, + "error": "", + "truncated": false, + "wallMs": 153022, + "loadMs": 3, + "outputTokens": 518, + "decodeMs": 132063.712, + "promptTokens": 483, + "prefillMs": 20827.197, + "thinkingChars": 0, + "answer": "```python\ndef parse_csv_line(line: str) -> list[str]:\n \"\"\"\n Parse a single CSV line into a list of fields without using the csv module.\n \n Rules:\n - Fields are separated by commas.\n - An empty line returns [\"\"].\n - Quoted fields use double quotes (\"). Inside quotes:\n - Commas are literals (not separators).\n - \"\" represents a single double quote character.\n - Unquoted fields are returned exactly as-is (no trimming).\n - Raises ValueError for unterminated quoted fields or invalid characters after a closing quote.\n \n :param line: The CSV line string to parse.\n :return: A list of field strings.\n :raises ValueError: If the CSV syntax is invalid.\n \"\"\"\n if not line:\n return [\"\"]\n \n fields = []\n current_field = []\n in_quotes = False\n \n i = 0\n n = len(line)\n \n while i < n:\n char = line[i]\n \n if in_quotes:\n if char == '\"':\n # Check if this is an escaped quote (\"\")\n if i + 1 < n and line[i + 1] == '\"':\n current_field.append('\"')\n i += 2 # Skip the next quote\n else:\n # End of quoted field\n in_quotes = False\n # Check if there's a comma immediately after the closing quote\n if i + 1 < n and line[i + 1] != ',':\n raise ValueError(\"Invalid character after closing quote\")\n i += 1 # Skip the comma\n else:\n current_field.append(char)\n continue\n \n else:\n if char == '\"':\n # Start of a quoted field\n in_quotes = True\n i += 1\n continue\n elif char == ',':\n # End of an unquoted field\n fields.append(''.join(current_field))\n current_field = []\n i += 1\n continue\n else:\n current_field.append(char)\n \n # After the loop, check for unterminated quoted field\n if in_quotes:\n raise ValueError(\"Unterminated quoted field\")\n \n # Add the last field\n fields.append(''.join(current_field))\n \n return fields\n```" + }, + { + "problem": "py-lcs", + "sample": 0, + "ok": false, + "error": "truncated by max tokens", + "truncated": true, + "wallMs": 539929, + "loadMs": 4, + "outputTokens": 2048, + "decodeMs": 521888.474, + "promptTokens": 428, + "prefillMs": 17900.259, + "thinkingChars": 0, + "answer": "```python\ndef longest_common_subsequence(a: str, b: str) -> str:\n \"\"\"\n Returns one longest common subsequence of strings a and b.\n \n Uses dynamic programming with space-optimized reconstruction.\n Handles strings up to 2000 characters efficiently (O(n*m) time, O(min(n,m)) space).\n \n Args:\n a: First input string\n b: Second input string\n \n Returns:\n A string representing one of the longest common subsequences.\n \"\"\"\n if not a or not b:\n return \"\"\n \n # Ensure 'a' is the shorter string for space optimization\n if len(a) > len(b):\n a, b = b, a\n \n n = len(a)\n m = len(b)\n \n # dp[j] will store the length of LCS of a[0:i] and b[0:j]\n # We use two rows (current and previous) to save space\n dp = [0] * (m + 1)\n prev_dp = [0] * (m + 1)\n \n # Compute LCS lengths\n for i in range(1, n + 1):\n for j in range(1, m + 1):\n if a[i - 1] == b[j - 1]:\n dp[j] = prev_dp[j - 1] + 1\n else:\n # dp[j] = max(dp[j], prev_dp[j])\n # Since we're overwriting dp, we need to be careful.\n # Actually, in standard space-optimized LCS:\n # dp[j] represents LCS(a[0:i], b[0:j])\n # When a[i-1] != b[j-1], dp[j] = max(dp[j] (which is LCS(a[0:i-1], b[0:j])), prev_dp[j])\n # But wait, dp[j] currently holds LCS(a[0:i-1], b[0:j]) from the previous iteration.\n # So we don't need to do anything if they don't match? \n # No, that's incorrect. We need to carry forward the value from the previous row.\n # Let's re-think:\n # dp[j] in current row i should be:\n # if a[i-1] == b[j-1]: prev_dp[j-1] + 1\n # else: max(dp[j], prev_dp[j])\n # But dp[j] already holds the value from prev_dp[j] (from row i-1) at the start of row i.\n # So if they don't match, we just keep dp[j] as is? \n # Actually, no. dp[j] at start of row i is LCS(a[0:i-1], b[0:j]).\n # prev_dp[j] is also LCS(a[0:i-1], b[0:j]).\n # So they are the same. We don't need to do anything.\n # BUT, we are overwriting dp[j]. So we need to make sure we don't lose the value.\n # The standard way is:\n # if match: dp[j] = prev_dp[j-1] + 1\n # else: dp[j] = dp[j] (which is already LCS(a[0:i-1], b[0:j]))\n # So we don't need an explicit else.\n pass\n \n # Move current row to previous row for next iteration\n prev_dp, dp = dp, prev_dp\n # Clear current row (which will be overwritten next iteration) but we need to preserve values from prev_dp\n # Actually, we can just swap and then reset dp to zeros? \n # No, because in the next iteration, we need dp to start as the previous row's values.\n # But we swapped, so dp now holds the old prev_dp (which was the previous row).\n # And prev_dp now holds the current row we just computed.\n # So we need to reset prev_dp (which is now the old dp) to zeros? \n # No, because in the next iteration, we will compute new values for dp (which is now the old prev_dp).\n # But we want dp to represent the current row being built.\n # Let's clarify:\n # Let dp be the row for current i (being built)\n # Let prev_dp be the row for i-1\n # At start of loop i:\n # dp is actually the row for i-1 (from previous swap)\n # prev_dp is the row for i-2 (or zeros for i=1)\n # We want to compute new values for dp (current row i)\n # So we should use prev_dp (which is i-1) to compute dp (which is i)\n # But after computing, we swap: dp becomes i, prev_dp becomes i-1 (for next iteration)\n # However, we need to reset the row that will become prev_dp for the next iteration? \n # Actually, no. Because in the next iteration, we will overwrite dp (which is now i) with i+1.\n # And we will use prev_dp (which is i) to compute i+1.\n # So we don't need to reset.\n # BUT, there's a problem: when we swap, the new prev_dp (which was old dp) has values from row i.\n # And we want to use it as the previous row for row i+1. That's correct.\n # However, we need to make sure that when we compute row i+1, we start with dp being a fresh row?\n # No, because we are going to overwrite dp entirely in the next iteration.\n # So we don't need to reset.\n # BUT, there's a catch: in the inner loop, we do:\n # if a[i-1] == b[j-1]: dp[j] = prev_dp[j-1] + 1\n # Here, prev_dp[j-1] is from row i-1, which is correct.\n # And dp[j] is being overwritten, so its old value doesn't matter.\n # So we don't need to reset.\n # However, we must ensure that prev_dp is correctly set.\n # After the inner loop, we swap: prev_dp, dp = dp, prev_dp\n # Now dp holds the row we just computed (row i), and prev_dp holds row i-1.\n # For the next iteration (i+1), we want:\n # prev_dp to be row i (which is now in dp)\n # dp to be row i (which we will overwrite)\n # But we swapped, so prev_dp is now row i (good), and dp is row i-1 (which we will overwrite).\n # So it's correct.\n # However, we need to reset the row that we are about to overwrite? \n # No, because we are going to overwrite every element of dp in the next iteration.\n # So it's fine.\n # BUT, there's a logical error: \n # In the inner loop, we do:\n # if a[i-1] == b[j-1]: dp[j] = prev_dp[j-1] + 1\n # else: we leave dp[j] as is? \n # But dp[j] currently holds the value from row i-1 (because we swapped at the end of previous iteration).\n # And we want dp[j] to hold the value for row i.\n # If they don't match, dp[j] should be max(dp[j], prev_dp[j])? \n # But dp[j] is from row i-1, and prev_dp[j] is also from row i-1? \n # No, after swap: \n # dp is the row we just computed (row i)\n # prev_dp is the row from i-1\n # Wait, I think I got the swap wrong.\n # Let me restart the logic.\n \n # Correct space-optimized LCS:\n # We use two rows: current (dp) and previous (prev_dp).\n # Initialize both to zeros.\n # For each character in a (row i from 1 to n):\n # For each character in b (col j from 1 to m):\n # if a[i-1] == b[j-1]: dp[j] = prev_dp[j-1] + 1\n # else: dp[j] = max(dp[j], prev_dp[j])\n # But note: at the start of row i, dp[j] holds the value from row i-1 (which is LCS(a[0:i-1], b[0:j])).\n # And prev_dp[j] also holds the value from row i-1? \n # Actually, no. We need to distinguish:\n # Let dp be the row for the current i (being built)\n # Let prev_dp be the row for i" + }, + { + "problem": "py-number-words", + "sample": 0, + "ok": false, + "error": "Traceback (most recent call last):\n File \"/tmp/flycoder-bench-lxY6tv/test_solution.py\", line 7, in \n assert number_to_words(n) == words, (n, number_to_words(n))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: (1000010, 'ten one million')\n", + "truncated": false, + "wallMs": 245143, + "loadMs": 3, + "outputTokens": 858, + "decodeMs": 220932.557, + "promptTokens": 538, + "prefillMs": 24045.656, + "thinkingChars": 0, + "answer": "```python\ndef number_to_words(n: int) -> str:\n \"\"\"\n Convert an integer to its English words representation.\n \n Valid range: 0 to 999,999,999,999 inclusive.\n - Lowercase words.\n - Single spaces between words.\n - Hyphens for compound numbers between 21 and 99.\n - No \"and\".\n - Scale words: thousand, million, billion.\n \n Raises ValueError if n is outside the valid range or not an integer.\n \"\"\"\n if not isinstance(n, int):\n raise ValueError(\"Input must be an integer.\")\n if n < 0 or n > 999_999_999_999:\n raise ValueError(\"Input must be between 0 and 999,999,999,999 inclusive.\")\n\n # Base words for 0-19\n under_20 = [\n \"zero\", \"one\", \"two\", \"three\", \"four\", \"five\", \"six\", \"seven\", \"eight\", \"nine\",\n \"ten\", \"eleven\", \"twelve\", \"thirteen\", \"fourteen\", \"fifteen\", \"sixteen\",\n \"seventeen\", \"eighteen\", \"nineteen\"\n ]\n\n # Tens: twenty, thirty, etc.\n tens = [\n \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", # 0-9 placeholders\n \"ten\", \"eleven\", \"twelve\", \"thirteen\", \"fourteen\", \"fifteen\", \"sixteen\",\n \"seventeen\", \"eighteen\", \"nineteen\"\n ]\n tens = [\n \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", \"\", # 0-9\n \"ten\", \"eleven\", \"twelve\", \"thirteen\", \"fourteen\", \"fifteen\", \"sixteen\",\n \"seventeen\", \"eighteen\", \"nineteen\"\n ]\n tens = [\"\", \"ten\", \"twenty\", \"thirty\", \"forty\", \"fifty\", \"sixty\", \"seventy\", \n \"eighty\", \"ninety\"]\n\n # Scale words\n scales = [\"\", \"thousand\", \"million\", \"billion\"]\n\n def convert_under_thousand(num: int) -> str:\n \"\"\"Converts a number from 0 to 999 into words.\"\"\"\n if num == 0:\n return \"\"\n \n words = []\n \n # Hundreds place\n if num >= 100:\n words.append(under_20[num // 100])\n words.append(\"hundred\")\n num %= 100\n \n # Tens and units\n if num > 0:\n if num < 20:\n words.append(under_20[num])\n else:\n ten = tens[num // 10]\n unit = under_20[num % 10]\n if num % 10 == 0:\n words.append(ten)\n else:\n words.append(f\"{ten}-{unit}\")\n \n return \" \".join(words)\n\n if n == 0:\n return \"zero\"\n\n parts = []\n scale_index = 0\n\n while n > 0:\n chunk = n % 1000\n if chunk > 0:\n chunk_words = convert_under_thousand(chunk)\n if scale_index == 0:\n parts.append(chunk_words)\n else:\n parts.append(f\"{chunk_words} {scales[scale_index]}\")\n n //= 1000\n scale_index += 1\n\n return \" \".join(parts)\n```" + }, + { + "problem": "py-dijkstra", + "sample": 0, + "ok": true, + "error": null, + "truncated": false, + "wallMs": 201710, + "loadMs": 3, + "outputTokens": 657, + "decodeMs": 179258.684, + "promptTokens": 505, + "prefillMs": 22282.489, + "thinkingChars": 0, + "answer": "```python\nimport heapq\nfrom typing import List, Tuple, Union\n\ndef shortest_distances(\n n: int,\n edges: List[Tuple[Union[int, float], Union[int, float], Union[int, float]]],\n source: Union[int, float]\n) -> List[float]:\n \"\"\"\n Computes shortest distances from a source node in a directed weighted graph.\n \n Args:\n n: Number of nodes (0 to n-1).\n edges: List of tuples (u, v, weight).\n source: Source node index.\n \n Returns:\n List of shortest distances from source to each node.\n \n Raises:\n ValueError: If source is invalid or any weight is negative.\n \"\"\"\n # Validate source\n if not isinstance(source, int) or source < 0 or source >= n:\n raise ValueError(f\"Source node {source} is not a valid node for a graph with {n} nodes (0..{n-1}).\")\n\n # Validate edges and build adjacency list\n # Using a dictionary for the adjacency list to handle potential hash collisions or custom objects if needed,\n # but for ints, a list of lists is faster. We'll use a list of lists.\n adj: List[List[Tuple[int, float]]] = [[] for _ in range(n)]\n \n for u, v, w in edges:\n # Validate node indices in edges\n if not isinstance(u, int) or not isinstance(v, int):\n raise ValueError(\"Node indices in edges must be integers.\")\n if u < 0 or u >= n:\n raise ValueError(f\"Node index {u} in edge is out of range [0, {n-1}).\")\n if v < 0 or v >= n:\n raise ValueError(f\"Node index {v} in edge is out of range [0, {n-1}).\")\n \n # Validate weight\n if not isinstance(w, (int, float)):\n raise ValueError(\"Edge weights must be numbers.\")\n \n # Check for negative weight\n if w < 0:\n raise ValueError(\"Edge weights must be non-negative.\")\n \n adj[u].append((v, w))\n \n # Dijkstra's Algorithm using a min-heap priority queue\n # Initialize distances\n distances = [float('inf')] * n\n distances[source] = 0.0\n \n # Priority queue stores tuples (distance, node)\n pq = [(0.0, source)]\n \n while pq:\n current_dist, u = heapq.heappop(pq)\n \n # If current distance is greater than already found shortest distance, skip\n if current_dist > distances[u]:\n continue\n \n for v, weight in adj[u]:\n new_dist = current_dist + weight\n if new_dist < distances[v]:\n distances[v] = new_dist\n heapq.heappush(pq, (new_dist, v))\n \n return distances\n```" } ] } diff --git a/tests/modelfile.test.mjs b/tests/modelfile.test.mjs index 034a8ea..4690172 100644 --- a/tests/modelfile.test.mjs +++ b/tests/modelfile.test.mjs @@ -2,6 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { execFileSync } from 'node:child_process'; +import { PROBLEMS } from '../bench/problems.mjs'; const read = file => fs.readFileSync(new URL(`../${file}`, import.meta.url), 'utf8'); // Minimal Modelfile reader: FROM, REQUIRES, PARAMETER, and the triple-quoted SYSTEM and MESSAGE blocks. @@ -66,15 +67,16 @@ test('flycoder0.3pro has its own agent workflow on top of the shared rules', () test("the pro example conversation shows code that passes its own tests", () => { const messages = parse(read(PRO.file)).messages; assert.deepEqual(messages.map(m => m.role), ['user', 'assistant']); + // An example that solves a bench problem would hand the model the answer and inflate its score. + for (const p of PROBLEMS) assert.ok(!messages[0].content.includes(p.entry), `the example must not be the bench problem ${p.id}`); assert.ok(!messages[1].content.includes('"""'), 'triple double quotes would end the MESSAGE block'); const [code, tests] = [...messages[1].content.matchAll(/```python\n([\s\S]*?)```/g)].map(m => m[1]); assert.match(tests, /pytest/); // Run the example with plain asserts (pytest is not needed): the valid and invalid cases of its tests. - const valid = [...tests.matchAll(/\("([^"]*)", (\d+)\)/g)].map(m => [m[1], Number(m[2])]); - const invalid = JSON.parse(tests.match(/"text", (\[[^\]]*\])/)[1]); + const [valid, invalid] = [...tests.matchAll(/"text", (\[[^\]]*\])/g)].map(m => JSON.parse(m[1].replace(/,\s*\]$/, ']'))); assert.ok(valid.length >= 5 && invalid.length >= 5); - const check = `${code}\nfor text, seconds in ${JSON.stringify(valid)}:\n assert parse_duration(text) == seconds, text\n` + - `for text in ${JSON.stringify(invalid)}:\n try:\n parse_duration(text)\n except ValueError:\n continue\n raise AssertionError(text)\nprint("ok")\n`; + const check = `${code}\nfor text in ${JSON.stringify(valid)}:\n assert is_valid_ipv4(text), text\n` + + `for text in ${JSON.stringify(invalid)}:\n assert not is_valid_ipv4(text), text\nprint("ok")\n`; assert.equal(execFileSync('python3', ['-c', check], { encoding: 'utf8' }).trim(), 'ok'); });