Spreadsheet Demo With Gemini (LLM)
Evaluation of Gemini (LLM) on a traditional coding interview question.
I recently tested Gemini, a large language model, on a simplified Python spreadsheet coding task, simulating a potential interview scenario. While the process revealed areas for improvement, Gemini’s performance was overall impressive.
- The final output was quite good. It had all the basics, like correctness, tests, caching, etc., in place.
- It was good at following guidance. It made many mistakes, but it was able to correct itself iteratively. However, this happened one issue at a time. At times, it was one step forward and one step backward, introducing new errors.
- The entire exercise took roughly 40 minutes, comparable to the time allotted in typical coding interviews.
- It could not develop the overall structure after the initial prompt. This result might be my mistake; perhaps I should have asked with good, open-ended prompts. Further exploration is needed to understand how effectively LLMs can handle complex software design tasks.
After all, if this were a coding interview, I would not be impressed, but this was quite impressive for supporting coding assistance.
If you are interested, check out the final code with some minor finishing touches from me here: https://gitlab.com/-/snippets/3679180.
spreadsheet.py
3.54 KiB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
from enum import Enum
import math
from typing import Tuple, Union
class CellType(Enum):
CONST = "CONST"
SUM = "SUM"
class Cell:
def __init__(self, cell_type: CellType, value: Union[float, Tuple[float, float]]):
self.cell_type = cell_type
if cell_type == CellType.CONST:
self.value = value
elif cell_type == CellType.SUM:
if not isinstance(value, tuple) or len(value) != 2:
raise ValueError("SUM cell must contain a tuple of two integer references")
self.references = value
else:
raise ValueError(f"Invalid cell type: {cell_type}")
def __str__(self):
if self.cell_type == CellType.CONST:
return str(self.value)
else:
return f"=({self.references[0]},{self.references[1]})"
def evaluate(self, spreadsheet) -> float:
# Check for CONST cells first (optimization)
if self.cell_type == CellType.CONST:
return self.value
# Check cache for previously calculated value
key = self
if key in spreadsheet.cache:
return spreadsheet.cache[key]
# Handle SUM cells with memoization
try:
ref1_value = spreadsheet.get_cell(self.references[0]).evaluate(spreadsheet)
ref2_value = spreadsheet.get_cell(self.references[1]).evaluate(spreadsheet)
result = ref1_value + ref2_value
except (IndexError, RecursionError):
# Handle both IndexError and RecursionError with NaN
result = math.nan
# Store evaluated value in cache for future use
spreadsheet.cache[key] = result
return result
class Spreadsheet:
def __init__(self):
self.cells = [Cell(CellType.CONST, 0) for _ in range(1000)] # Initialize all cells with 0s
self.cache = {}
def set_cell(self, index: int, cell: Cell) -> None:
if not isinstance(cell, Cell):
raise ValueError("Cell must be an instance of the Cell class")
if index < 0 or index >= 1000:
raise IndexError("Index out of bounds")
self.cells[index] = cell
def get_cell(self, index: int) -> float:
if index < 0 or index >= 1000:
raise IndexError("Index out of bounds")
return self.cells[index]
def evaluate(self) -> None:
# Clear the cache for a fresh evaluation
self.cache = {}
# Evaluate all cells in the spreadsheet
for cell in self.cells:
if cell is not None:
cell.evaluate(self) # Pass the spreadsheet instance for memoization
# Example usage
spreadsheet = Spreadsheet()
spreadsheet.set_cell(0, Cell(CellType.CONST, 5)) # Cell 0
spreadsheet.set_cell(1, Cell(CellType.CONST, 10)) # Cell 1
spreadsheet.set_cell(2, Cell(CellType.SUM, (0, 1))) # Cell 2 sums cell 0 and cell 1
spreadsheet.set_cell(3, Cell(CellType.SUM, (4, 0))) # Cell 3 (circular reference)
spreadsheet.set_cell(4, Cell(CellType.SUM, (3, 0))) # Cell 4 (circular reference)
spreadsheet.set_cell(5, Cell(CellType.SUM, (1000, 1001))) # Cell 5 with out of bounds references
print(spreadsheet.get_cell(0).evaluate(spreadsheet)) # Output: 5
print(spreadsheet.get_cell(1).evaluate(spreadsheet)) # Output: 10
print(spreadsheet.get_cell(2).evaluate(spreadsheet)) # Output: 15
print(spreadsheet.get_cell(3).evaluate(spreadsheet)) # Output: nan (circular reference)
print(spreadsheet.get_cell(4).evaluate(spreadsheet)) # Output: nan (circular reference)
print(spreadsheet.get_cell(5).evaluate(spreadsheet)) # Output: nan (out of bounds reference)