🐛 use 512 tokens instead of 256 (#509)

# Description

Recent updates are causing issues with running granite models @ context
length 256. This updates the unit tests to default to 512 instead.

---------

Signed-off-by: Joe Runde <[email protected]>

Author: joerunde

tests/conftest.py

@@ -31,7 +31,7 @@ def pytest_generate_tests(metafunc):
     # default parameterizations
     default_warmup_shape = [[(64, 20, 4)]]
     default_max_num_seqs = [4]
-    default_max_model_len = [256]
+    default_max_model_len = [512]
 
     existing_markers = [
         marker.name if marker.name != "parametrize" else marker.args[0]

tests/e2e/test_spyre_basic.py

@@ -137,7 +137,7 @@ def test_batch_handling(model: ModelInfo, backend: str, cb: int, warmup_shapes,
     vllm_results = generate_spyre_vllm_output(
         model=model,
         prompts=prompts,
-        max_model_len=256,
+        max_model_len=max_model_len,
         sampling_params=vllm_sampling_params,
         tensor_parallel_size=1,
         backend=backend,

tests/llm_cache.py

@@ -180,7 +180,7 @@ def get_engine(
 
         # 🌶️🌶️🌶️
         # Messing with the blocks and context length by either:
-        # - setting context < 256 tokens
+        # - setting context < 512 tokens
         # - setting available blocks != (context * batch size // 64)
         # can cause compilation failures on spyre hardware.
 
@@ -195,7 +195,7 @@ def get_engine(
         engine_args = EngineArgs(
             model=model_name,
             tokenizer=model_name,
-            max_model_len=max(max_model_len, 256),
+            max_model_len=max(max_model_len, 512),
             max_num_seqs=max_num_seqs_compiled,
             num_gpu_blocks_override=None,
             revision=revision,