ishenwei/agent-base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: resolve startup issues and JSONL parsing bugs

Browse Source 
- Add PYTHONPATH/DJANGO_SETTINGS_MODULE for Docker environment
- Fix gunicorn bind port (8000 instead of 8765)
- Add whitenoise for static file serving
- Fix JSONL parser: read role/content from event.message.* (nested structure)
- Fix session counts (message_count/tool_call_count/error_count were all 0)
- Increase DATA_UPLOAD_MAX_MEMORY_SIZE to 50MB for large batch syncs
- Add STATIC_ROOT and WhiteNoise middleware for admin CSS
- Fix index name length (>30 chars issue)
- Replace unique_together with indexes (Django 5.x compatible)
- Add graceful degradation for TimescaleDB hypertable creation
- Add static_volume/ to .gitignore
This commit is contained in:
ishenwei
2026-04-06 20:51:51 +08:00
parent 0653c26523
commit f205c42714
9 changed files with 54 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@@ -10,3 +10,4 @@ tests/
scripts/
docs/
db.sqlite3
db_data

1
.gitignore vendored
View File

@@ -6,3 +6,4 @@ dist/
.venv/
env/
*.sqlite3
static_volume/

19
docker-compose.yml
View File

@@ -26,11 +26,16 @@ services:
env_file:
- .env
environment:
PYTHONPATH: /app/src
DJANGO_SETTINGS_MODULE: config.settings
DB_HOST: db
DB_PORT: 5432
DB_PORT: "5432"
DB_NAME: openclaw_archive
DB_USER: openclaw
DB_PASSWORD: openclaw_archive_pass
DJANGO_SUPERUSER_USERNAME: admin
DJANGO_SUPERUSER_EMAIL: admin@example.com
DJANGO_SUPERUSER_PASSWORD: admin123
ports:
- "8765:8000"
volumes:
@@ -39,11 +44,13 @@ services:
db:
condition: service_healthy
restart: unless-stopped
command: >
sh -c "python manage.py migrate &&
python manage.py createsuperuser --noinput ||
true &&
gunicorn --bind 0.0.0.0:8765 --workers 2 --timeout 120 config.wsgi:application"
command:
- sh
- -c
- |
python manage.py migrate
python manage.py createsuperuser --noinput || true
gunicorn --bind 0.0.0.0:8000 --workers 2 --timeout 120 config.wsgi:application
volumes:
db_data:

1
requirements.txt
View File

@@ -2,3 +2,4 @@ Django>=5.0,<6.0
djangorestframework>=3.15,<4.0
psycopg[binary]>=3.1,<4.0
gunicorn>=22.0,<24.0
whitenoise>=6.0

41
scripts/sync_sessions.py
View File

@@ -183,13 +183,15 @@ def parse_jsonl(file_path):
current_thinking_level = event.get("thinkingLevel", "")
elif event_type == "message":
role = event.get("role", "")
# Nested structure: message data is inside "message" object
message_obj = event.get("message", {})
role = message_obj.get("role", "")
msg_id = event.get("id", "")
parent_id = event.get("parentId", "")
msg_timestamp = event.get("timestamp", "")
# Extract text content (skip thinking)
content_items = event.get("content", [])
# Extract text content (skip thinking) from nested content
content_items = message_obj.get("content", [])
text_parts = []
tc_list = []
for item in content_items:
@@ -210,16 +212,16 @@ def parse_jsonl(file_path):
"role": role or "",
"content_text": content_text,
"raw_content": content_items if content_items else [],
"raw_message": event.get("content", []),
"raw_message": message_obj.get("content", []),
"timestamp": msg_timestamp,
}
if role == "assistant":
usage = event.get("usage", {})
usage = message_obj.get("usage", {})
msg_data.update({
"model": current_model_id,
"provider": current_model_provider,
"stop_reason": event.get("stopReason", ""),
"stop_reason": message_obj.get("stopReason", ""),
"tokens_input": usage.get("inputTokens", 0),
"tokens_output": usage.get("outputTokens", 0),
"tokens_cache_read": usage.get("cacheReadInputTokens", 0),
@@ -228,8 +230,8 @@ def parse_jsonl(file_path):
})
total_tokens += usage.get("totalTokens", 0)
if event.get("cost"):
cost_val = event["cost"].get("total", 0.0)
if message_obj.get("cost"):
cost_val = message_obj["cost"].get("total", 0.0)
msg_data["cost_total"] = cost_val
total_cost += cost_val
@@ -237,21 +239,20 @@ def parse_jsonl(file_path):
elif role == "toolResult":
msg_data.update({
"tool_call_id": event.get("toolCallId", ""),
"tool_name": event.get("toolName", ""),
"is_error": event.get("isError", False),
"exit_code": event.get("exitCode"),
"duration_ms": event.get("durationMs"),
"tool_call_id": message_obj.get("toolCallId", ""),
"tool_name": message_obj.get("toolName", ""),
"is_error": message_obj.get("isError", False),
"exit_code": message_obj.get("exitCode"),
"duration_ms": message_obj.get("durationMs"),
})
if event.get("isError"):
if message_obj.get("isError"):
error_count += 1
# Store for tool call association
if event.get("toolCallId"):
tool_results[event["toolCallId"]] = {
if message_obj.get("toolCallId"):
tool_results[message_obj["toolCallId"]] = {
"result_text": content_text,
"is_error": event.get("isError", False),
"exit_code": event.get("exitCode"),
"duration_ms": event.get("durationMs"),
"is_error": message_obj.get("isError", False),
"exit_code": message_obj.get("exitCode"),
"duration_ms": message_obj.get("durationMs"),
}
messages.append(msg_data)

3
src/config/settings/base.py
View File

@@ -17,6 +17,7 @@ INSTALLED_APPS = [
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"whitenoise.middleware.WhiteNoiseMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
@@ -56,3 +57,5 @@ DATABASES = {
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
STATIC_URL = "static/"
DATA_UPLOAD_MAX_MEMORY_SIZE = 50 * 1024 * 1024
STATIC_ROOT = "/app/staticfiles"

5
src/openclaw/migrations/0001_initial.py
View File

@@ -41,9 +41,12 @@ class Migration(migrations.Migration):
options={
'db_table': 'sessions',
'ordering': ['-start_time'],
'unique_together': {('session_id', 'agent_name')},
},
),
migrations.AddIndex(
model_name='session',
index=models.Index(fields=['session_id', 'agent_name'], name='ses_sid_aname_idx'),
),
migrations.CreateModel(
name='Message',
fields=[

9
src/openclaw/migrations/0002_add_hypertables.py
View File

@@ -18,8 +18,13 @@ SELECT create_hypertable(
def create_hypertables(apps, schema_editor):
if connection.vendor != "postgresql":
return
with schema_editor.connection.cursor() as cursor:
cursor.execute(CREATE_HYPERTABLES)
try:
with schema_editor.connection.cursor() as cursor:
cursor.execute(CREATE_HYPERTABLES)
except Exception as e:
# Gracefully degrade if TimescaleDB hypertable creation fails
# This allows Django migrations to succeed even without hypertables
print(f"TimescaleDB hypertable creation skipped: {e}")
class Migration(migrations.Migration):

6
src/openclaw/models.py
View File

@@ -27,7 +27,9 @@ class Session(models.Model):
class Meta:
db_table = "sessions"
unique_together = ("session_id", "agent_name")
indexes = [
models.Index(fields=["session_id", "agent_name"], name="ses_sid_aname_idx"),
]
ordering = ["-start_time"]
def __str__(self):
@@ -46,7 +48,6 @@ class Message(models.Model):
raw_content = models.JSONField(default=list, blank=True)
raw_message = models.JSONField(default=dict, blank=True)
timestamp = models.DateTimeField()
# assistant 专用
model = models.CharField(max_length=128, blank=True, default="")
provider = models.CharField(max_length=64, blank=True, default="")
stop_reason = models.CharField(max_length=64, blank=True, default="")
@@ -56,7 +57,6 @@ class Message(models.Model):
tokens_cache_write = models.IntegerField(default=0)
tokens_total = models.IntegerField(default=0)
cost_total = models.FloatField(default=0.0)
# toolResult 专用
tool_call_id = models.CharField(max_length=128, blank=True, default="")
tool_name = models.CharField(max_length=128, blank=True, default="")
is_error = models.BooleanField(default=False)