cPhulkd Investigation & Fix Notes¶

Date: 2026-05-03
Server: vps-60b0ec50

Problem Summary¶

cPhulkd DB processor repeatedly cycling (starting and dying via SIGTERM ~every 10–20 min)
Timeout errors in pre-auth state
Root cause: ip_lists table bloated to 599,508 rows causing slow full-range scans on every auth attempt

Diagnostic Commands¶

1. Check cPhulkd Logs¶

tail -f /usr/local/cpanel/logs/cphulkd.log

2. MySQL Health Check¶

mysqladmin status
# Red flag: Slow queries / Questions ratio > 50%

3. SQLite Integrity Check¶

cd /var/cpanel/hulkd
sqlite3 cphulk.sqlite "PRAGMA integrity_check;"

4. Check Table Row Counts¶

python3 - "/var/cpanel/hulkd/cphulk.sqlite" <<'PY'
import sqlite3, sys
db = sys.argv[1]
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=30)
tables = [r[0] for r in con.execute(
    "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
)]
for t in tables:
    safe = '"' + t.replace('"', '""') + '"'
    count = con.execute(f"SELECT COUNT(*) FROM {safe}").fetchone()[0]
    print(f"{t:<45} {count}")
con.close()
PY

5. Check Indexes on ip_lists¶

python3 - "/var/cpanel/hulkd/cphulk.sqlite" <<'PY'
import sqlite3, sys
con = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
for row in con.execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name='ip_lists'"):
    print(row)
PY

6. Inspect Schema and Sample Rows¶

python3 - "/var/cpanel/hulkd/cphulk.sqlite" <<'PY'
import sqlite3, sys
con = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)

print("=== TABLE SCHEMA ===")
for row in con.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='ip_lists'"):
    print(row[0])

print("\n=== INDEX DETAILS ===")
for row in con.execute("PRAGMA index_list('ip_lists')"):
    print("Index:", row)
    idx_name = row[1]
    for col in con.execute(f"PRAGMA index_info('{idx_name}')"):
        print("   ", col)
PY

7. Decode Binary IP Data and Type Breakdown¶

python3 - "/var/cpanel/hulkd/cphulk.sqlite" <<'PY'
import sqlite3, sys, socket

con = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
con.text_factory = bytes

def decode_ip(b):
    if b is None: return "NULL"
    if len(b) == 4:  return socket.inet_ntop(socket.AF_INET, b)
    if len(b) == 16: return socket.inet_ntop(socket.AF_INET6, b)
    return b.hex()

print("=== TYPE BREAKDOWN ===")
for row in con.execute("SELECT TYPE, COUNT(*) FROM ip_lists GROUP BY TYPE ORDER BY 2 DESC"):
    print(f"  TYPE={row[0]}  count={row[1]}")

print("\n=== SAMPLE ROWS ===")
for row in con.execute("SELECT STARTADDRESS, ENDADDRESS, TYPE, COMMENT FROM ip_lists LIMIT 10"):
    print(f"  {decode_ip(row[0]):<40} -> {decode_ip(row[1]):<40}  TYPE={row[2]}  {row[3]}")
PY

8. Backup Whitelisted IPs Before Wiping (TYPE=1)¶

python3 - "/var/cpanel/hulkd/cphulk.sqlite" <<'PY'
import sqlite3, sys, socket
con = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
con.text_factory = bytes
def decode_ip(b):
    if len(b) == 4: return socket.inet_ntop(socket.AF_INET, b)
    if len(b) == 16: return socket.inet_ntop(socket.AF_INET6, b)
    return b.hex()
for row in con.execute("SELECT STARTADDRESS, ENDADDRESS, COMMENT FROM ip_lists WHERE TYPE=1"):
    print(decode_ip(row[0]), '->', decode_ip(row[1]), row[3])
PY

Fix: Rebuild the cPhulkd Database¶

# 1. Stop cPhulkd
/usr/local/cpanel/scripts/restartsrv_cphulkd stop

# 2. Backup and remove bloated DB
cp /var/cpanel/hulkd/cphulk.sqlite /var/cpanel/hulkd/cphulk.sqlite.bak.$(date +%Y%m%d)
rm /var/cpanel/hulkd/cphulk.sqlite

# 3. Start cPhulkd — rebuilds DB automatically
/usr/local/cpanel/scripts/restartsrv_cphulkd start

# 4. Monitor logs for clean startup
tail -f /usr/local/cpanel/logs/cphulkd.log

Verify After Fix¶

python3 - "/var/cpanel/hulkd/cphulk.sqlite" <<'PY'
import sqlite3, sys
con = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
tables = [r[0] for r in con.execute("SELECT name FROM sqlite_master WHERE type='table'")]
for t in tables:
    count = con.execute(f'SELECT COUNT(*) FROM "{t}"').fetchone()[0]
    print(f"{t:<45} {count}")
PY

ip_lists should now show a small row count.

ip_lists TYPE Reference¶

TYPE	Meaning
0	Blacklisted IP/range
1	Whitelisted IP/range
2	Country block netblocks

Notes¶

Bloat is typically caused by large country blacklists in cPhulk being expanded into individual CIDR ranges
The UNIQUE index on (STARTADDRESS, ENDADDRESS) is correct but insufficient for range-based lookups at 500k+ rows
After rebuild, monitor ip_lists row count periodically — if it bloats again, consider reducing the number of blacklisted countries in WHM → cPhulk settings