fix: resolve startup issues and JSONL parsing bugs
- Add PYTHONPATH/DJANGO_SETTINGS_MODULE for Docker environment - Fix gunicorn bind port (8000 instead of 8765) - Add whitenoise for static file serving - Fix JSONL parser: read role/content from event.message.* (nested structure) - Fix session counts (message_count/tool_call_count/error_count were all 0) - Increase DATA_UPLOAD_MAX_MEMORY_SIZE to 50MB for large batch syncs - Add STATIC_ROOT and WhiteNoise middleware for admin CSS - Fix index name length (>30 chars issue) - Replace unique_together with indexes (Django 5.x compatible) - Add graceful degradation for TimescaleDB hypertable creation - Add static_volume/ to .gitignore
This commit is contained in:
@@ -10,3 +10,4 @@ tests/
|
|||||||
scripts/
|
scripts/
|
||||||
docs/
|
docs/
|
||||||
db.sqlite3
|
db.sqlite3
|
||||||
|
db_data
|
||||||
|
|||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -6,3 +6,4 @@ dist/
|
|||||||
.venv/
|
.venv/
|
||||||
env/
|
env/
|
||||||
*.sqlite3
|
*.sqlite3
|
||||||
|
static_volume/
|
||||||
|
|||||||
@@ -26,11 +26,16 @@ services:
|
|||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
environment:
|
environment:
|
||||||
|
PYTHONPATH: /app/src
|
||||||
|
DJANGO_SETTINGS_MODULE: config.settings
|
||||||
DB_HOST: db
|
DB_HOST: db
|
||||||
DB_PORT: 5432
|
DB_PORT: "5432"
|
||||||
DB_NAME: openclaw_archive
|
DB_NAME: openclaw_archive
|
||||||
DB_USER: openclaw
|
DB_USER: openclaw
|
||||||
DB_PASSWORD: openclaw_archive_pass
|
DB_PASSWORD: openclaw_archive_pass
|
||||||
|
DJANGO_SUPERUSER_USERNAME: admin
|
||||||
|
DJANGO_SUPERUSER_EMAIL: admin@example.com
|
||||||
|
DJANGO_SUPERUSER_PASSWORD: admin123
|
||||||
ports:
|
ports:
|
||||||
- "8765:8000"
|
- "8765:8000"
|
||||||
volumes:
|
volumes:
|
||||||
@@ -39,11 +44,13 @@ services:
|
|||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: >
|
command:
|
||||||
sh -c "python manage.py migrate &&
|
- sh
|
||||||
python manage.py createsuperuser --noinput ||
|
- -c
|
||||||
true &&
|
- |
|
||||||
gunicorn --bind 0.0.0.0:8765 --workers 2 --timeout 120 config.wsgi:application"
|
python manage.py migrate
|
||||||
|
python manage.py createsuperuser --noinput || true
|
||||||
|
gunicorn --bind 0.0.0.0:8000 --workers 2 --timeout 120 config.wsgi:application
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
db_data:
|
db_data:
|
||||||
|
|||||||
@@ -2,3 +2,4 @@ Django>=5.0,<6.0
|
|||||||
djangorestframework>=3.15,<4.0
|
djangorestframework>=3.15,<4.0
|
||||||
psycopg[binary]>=3.1,<4.0
|
psycopg[binary]>=3.1,<4.0
|
||||||
gunicorn>=22.0,<24.0
|
gunicorn>=22.0,<24.0
|
||||||
|
whitenoise>=6.0
|
||||||
|
|||||||
@@ -183,13 +183,15 @@ def parse_jsonl(file_path):
|
|||||||
current_thinking_level = event.get("thinkingLevel", "")
|
current_thinking_level = event.get("thinkingLevel", "")
|
||||||
|
|
||||||
elif event_type == "message":
|
elif event_type == "message":
|
||||||
role = event.get("role", "")
|
# Nested structure: message data is inside "message" object
|
||||||
|
message_obj = event.get("message", {})
|
||||||
|
role = message_obj.get("role", "")
|
||||||
msg_id = event.get("id", "")
|
msg_id = event.get("id", "")
|
||||||
parent_id = event.get("parentId", "")
|
parent_id = event.get("parentId", "")
|
||||||
msg_timestamp = event.get("timestamp", "")
|
msg_timestamp = event.get("timestamp", "")
|
||||||
|
|
||||||
# Extract text content (skip thinking)
|
# Extract text content (skip thinking) from nested content
|
||||||
content_items = event.get("content", [])
|
content_items = message_obj.get("content", [])
|
||||||
text_parts = []
|
text_parts = []
|
||||||
tc_list = []
|
tc_list = []
|
||||||
for item in content_items:
|
for item in content_items:
|
||||||
@@ -210,16 +212,16 @@ def parse_jsonl(file_path):
|
|||||||
"role": role or "",
|
"role": role or "",
|
||||||
"content_text": content_text,
|
"content_text": content_text,
|
||||||
"raw_content": content_items if content_items else [],
|
"raw_content": content_items if content_items else [],
|
||||||
"raw_message": event.get("content", []),
|
"raw_message": message_obj.get("content", []),
|
||||||
"timestamp": msg_timestamp,
|
"timestamp": msg_timestamp,
|
||||||
}
|
}
|
||||||
|
|
||||||
if role == "assistant":
|
if role == "assistant":
|
||||||
usage = event.get("usage", {})
|
usage = message_obj.get("usage", {})
|
||||||
msg_data.update({
|
msg_data.update({
|
||||||
"model": current_model_id,
|
"model": current_model_id,
|
||||||
"provider": current_model_provider,
|
"provider": current_model_provider,
|
||||||
"stop_reason": event.get("stopReason", ""),
|
"stop_reason": message_obj.get("stopReason", ""),
|
||||||
"tokens_input": usage.get("inputTokens", 0),
|
"tokens_input": usage.get("inputTokens", 0),
|
||||||
"tokens_output": usage.get("outputTokens", 0),
|
"tokens_output": usage.get("outputTokens", 0),
|
||||||
"tokens_cache_read": usage.get("cacheReadInputTokens", 0),
|
"tokens_cache_read": usage.get("cacheReadInputTokens", 0),
|
||||||
@@ -228,8 +230,8 @@ def parse_jsonl(file_path):
|
|||||||
})
|
})
|
||||||
total_tokens += usage.get("totalTokens", 0)
|
total_tokens += usage.get("totalTokens", 0)
|
||||||
|
|
||||||
if event.get("cost"):
|
if message_obj.get("cost"):
|
||||||
cost_val = event["cost"].get("total", 0.0)
|
cost_val = message_obj["cost"].get("total", 0.0)
|
||||||
msg_data["cost_total"] = cost_val
|
msg_data["cost_total"] = cost_val
|
||||||
total_cost += cost_val
|
total_cost += cost_val
|
||||||
|
|
||||||
@@ -237,21 +239,20 @@ def parse_jsonl(file_path):
|
|||||||
|
|
||||||
elif role == "toolResult":
|
elif role == "toolResult":
|
||||||
msg_data.update({
|
msg_data.update({
|
||||||
"tool_call_id": event.get("toolCallId", ""),
|
"tool_call_id": message_obj.get("toolCallId", ""),
|
||||||
"tool_name": event.get("toolName", ""),
|
"tool_name": message_obj.get("toolName", ""),
|
||||||
"is_error": event.get("isError", False),
|
"is_error": message_obj.get("isError", False),
|
||||||
"exit_code": event.get("exitCode"),
|
"exit_code": message_obj.get("exitCode"),
|
||||||
"duration_ms": event.get("durationMs"),
|
"duration_ms": message_obj.get("durationMs"),
|
||||||
})
|
})
|
||||||
if event.get("isError"):
|
if message_obj.get("isError"):
|
||||||
error_count += 1
|
error_count += 1
|
||||||
# Store for tool call association
|
if message_obj.get("toolCallId"):
|
||||||
if event.get("toolCallId"):
|
tool_results[message_obj["toolCallId"]] = {
|
||||||
tool_results[event["toolCallId"]] = {
|
|
||||||
"result_text": content_text,
|
"result_text": content_text,
|
||||||
"is_error": event.get("isError", False),
|
"is_error": message_obj.get("isError", False),
|
||||||
"exit_code": event.get("exitCode"),
|
"exit_code": message_obj.get("exitCode"),
|
||||||
"duration_ms": event.get("durationMs"),
|
"duration_ms": message_obj.get("durationMs"),
|
||||||
}
|
}
|
||||||
|
|
||||||
messages.append(msg_data)
|
messages.append(msg_data)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ INSTALLED_APPS = [
|
|||||||
|
|
||||||
MIDDLEWARE = [
|
MIDDLEWARE = [
|
||||||
"django.middleware.security.SecurityMiddleware",
|
"django.middleware.security.SecurityMiddleware",
|
||||||
|
"whitenoise.middleware.WhiteNoiseMiddleware",
|
||||||
"django.contrib.sessions.middleware.SessionMiddleware",
|
"django.contrib.sessions.middleware.SessionMiddleware",
|
||||||
"django.middleware.common.CommonMiddleware",
|
"django.middleware.common.CommonMiddleware",
|
||||||
"django.middleware.csrf.CsrfViewMiddleware",
|
"django.middleware.csrf.CsrfViewMiddleware",
|
||||||
@@ -56,3 +57,5 @@ DATABASES = {
|
|||||||
|
|
||||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||||
STATIC_URL = "static/"
|
STATIC_URL = "static/"
|
||||||
|
DATA_UPLOAD_MAX_MEMORY_SIZE = 50 * 1024 * 1024
|
||||||
|
STATIC_ROOT = "/app/staticfiles"
|
||||||
|
|||||||
@@ -41,9 +41,12 @@ class Migration(migrations.Migration):
|
|||||||
options={
|
options={
|
||||||
'db_table': 'sessions',
|
'db_table': 'sessions',
|
||||||
'ordering': ['-start_time'],
|
'ordering': ['-start_time'],
|
||||||
'unique_together': {('session_id', 'agent_name')},
|
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
migrations.AddIndex(
|
||||||
|
model_name='session',
|
||||||
|
index=models.Index(fields=['session_id', 'agent_name'], name='ses_sid_aname_idx'),
|
||||||
|
),
|
||||||
migrations.CreateModel(
|
migrations.CreateModel(
|
||||||
name='Message',
|
name='Message',
|
||||||
fields=[
|
fields=[
|
||||||
|
|||||||
@@ -18,8 +18,13 @@ SELECT create_hypertable(
|
|||||||
def create_hypertables(apps, schema_editor):
|
def create_hypertables(apps, schema_editor):
|
||||||
if connection.vendor != "postgresql":
|
if connection.vendor != "postgresql":
|
||||||
return
|
return
|
||||||
|
try:
|
||||||
with schema_editor.connection.cursor() as cursor:
|
with schema_editor.connection.cursor() as cursor:
|
||||||
cursor.execute(CREATE_HYPERTABLES)
|
cursor.execute(CREATE_HYPERTABLES)
|
||||||
|
except Exception as e:
|
||||||
|
# Gracefully degrade if TimescaleDB hypertable creation fails
|
||||||
|
# This allows Django migrations to succeed even without hypertables
|
||||||
|
print(f"TimescaleDB hypertable creation skipped: {e}")
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
class Migration(migrations.Migration):
|
||||||
|
|||||||
@@ -27,7 +27,9 @@ class Session(models.Model):
|
|||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
db_table = "sessions"
|
db_table = "sessions"
|
||||||
unique_together = ("session_id", "agent_name")
|
indexes = [
|
||||||
|
models.Index(fields=["session_id", "agent_name"], name="ses_sid_aname_idx"),
|
||||||
|
]
|
||||||
ordering = ["-start_time"]
|
ordering = ["-start_time"]
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
@@ -46,7 +48,6 @@ class Message(models.Model):
|
|||||||
raw_content = models.JSONField(default=list, blank=True)
|
raw_content = models.JSONField(default=list, blank=True)
|
||||||
raw_message = models.JSONField(default=dict, blank=True)
|
raw_message = models.JSONField(default=dict, blank=True)
|
||||||
timestamp = models.DateTimeField()
|
timestamp = models.DateTimeField()
|
||||||
# assistant 专用
|
|
||||||
model = models.CharField(max_length=128, blank=True, default="")
|
model = models.CharField(max_length=128, blank=True, default="")
|
||||||
provider = models.CharField(max_length=64, blank=True, default="")
|
provider = models.CharField(max_length=64, blank=True, default="")
|
||||||
stop_reason = models.CharField(max_length=64, blank=True, default="")
|
stop_reason = models.CharField(max_length=64, blank=True, default="")
|
||||||
@@ -56,7 +57,6 @@ class Message(models.Model):
|
|||||||
tokens_cache_write = models.IntegerField(default=0)
|
tokens_cache_write = models.IntegerField(default=0)
|
||||||
tokens_total = models.IntegerField(default=0)
|
tokens_total = models.IntegerField(default=0)
|
||||||
cost_total = models.FloatField(default=0.0)
|
cost_total = models.FloatField(default=0.0)
|
||||||
# toolResult 专用
|
|
||||||
tool_call_id = models.CharField(max_length=128, blank=True, default="")
|
tool_call_id = models.CharField(max_length=128, blank=True, default="")
|
||||||
tool_name = models.CharField(max_length=128, blank=True, default="")
|
tool_name = models.CharField(max_length=128, blank=True, default="")
|
||||||
is_error = models.BooleanField(default=False)
|
is_error = models.BooleanField(default=False)
|
||||||
|
|||||||
Reference in New Issue
Block a user