H2 workspace related panic #4396
Description
Originally posted by @desdic in #4368 (comment)
The server was running the master branch and not #4368
Child (23116) Panic at: Sat, 13 Sep 2025 17:08:11 GMT
Assert error in h2_del_sess(), http2/cache_http2_session.c line 185:
Condition(!WS_IsReserved(req->ws)) not true.
version = varnish-trunk revision 43fbfd6051f0ea3760d04b893ebebcb5351ad263, vrt api = 21.0
ident = Linux,6.8.0-51-generic,x86_64,-jlinux,-smalloc,-sdefault,-hcritbit,epoll
now = 11612746.444027 (mono), 1757783219.043331 (real)
Backtrace:
ip=0x5931b56e2ab5 sp=0x734bb316a300 <VBT_format+0x65>
ip=0x5931b56562cd sp=0x734bb316a420 <pan_ic+0x24d>
ip=0x5931b56e1d39 sp=0x734bb316a580 <VAS_Fail+0x19>
ip=0x5931b569e19b sp=0x734bb316a590 <h2_del_sess+0x11b>
ip=0x5931b569ed37 sp=0x734bb316a5c0 <h2_new_session+0x8a7>
ip=0x5931b5683122 sp=0x734bb316a870 <WRK_Thread+0x3c2>
ip=0x5931b568370c sp=0x734bb316b420 <pool_thread+0x3c>
ip=0x734bb329caa4 sp=0x734bb316b450 <pthread_condattr_setpshared+0x684>
ip=0x734bb3329c3c sp=0x734bb316b500 <__clone+0x24c>
errno = 110 (Connection timed out)
argv = {
[0] = \"/usr/sbin/varnishd\",
[1] = \"-a\",
[2] = \":80\",
[3] = \"-a\",
[4] = \"127.0.0.1:81,PROXY\",
[5] = \"-a\",
[6] = \"/var/run/varnish.sock,PROXY,user=vcache,group=varnish,mode=666\",
[7] = \"-f\",
[8] = \"/etc/varnish/onecom.vcl\",
[9] = \"-T\",
[10] = \":6082\",
[11] = \"-t\",
[12] = \"120\",
[13] = \"-l\",
[14] = \"512M\",
[15] = \"-s\",
[16] = \"malloc,735G\",
[17] = \"-S\",
[18] = \"/etc/varnish/secret\",
[19] = \"-p\",
[20] = \"vcc_err_unref=false\",
[21] = \"-p\",
[22] = \"thread_pool_stack=192k\",
[23] = \"-p\",
[24] = \"workspace_client=128k\",
[25] = \"-p\",
[26] = \"workspace_backend=128k\",
[27] = \"-p\",
[28] = \"send_timeout=3600\",
[29] = \"-p\",
[30] = \"transit_buffer=1M\",
[31] = \"-p\",
[32] = \"h2_window_timeout=200\",
[33] = \"-p\",
[34] = \"timeout_idle=60\",
[35] = \"-p\",
[36] = \"vsl_mask=-H2TxHdr,-H2TxBody\",
[37] = \"-p\",
[38] = \"feature=+http2\",
[39] = \"-p\",
[40] = \"thread_pool_max=8000\",
[41] = \"-p\",
[42] = \"http_max_hdr=128\",
}
pthread.self = 0x734bb316c6c0
pthread.name = (cache-worker)
pthread.attr = {
guard = 4096,
stack_bottom = 0x734bb313d000,
stack_top = 0x734bb316d000,
stack_size = 196608,
}
thr.req = 0x7341998453e0 {
vxid = 1017251858, transport = HTTP/2
step = Req Step transport
req_body = NULL,
err_code = 1, err_reason = (null),
restarts = 0, esi_level = 0,
vary_b = (nil), vary_e = (nil),
d_ttl = 0.000000, d_grace = 0.000000,
storage = (nil),
sess = 0x727f88caea20 {
fd = 216, vxid = 1017251858,
t_open = 1757783218.307428,
t_idle = 1757783219.043316,
ws = 0x727f88caea60 {
id = \"ses\",
{s, f, r, e} = {0x727f88caeab8, +168, (nil), +576},
},
transport = HTTP/2 {
h2_sess = 0x734bb316a660 {
refcnt = 0, bogosity = 0, error = H2CE_RAPID_RESET
open_streams = 0, highest_stream = 113, goaway_last_stream = 0,
local_settings = {0x1000, 0x1, 0x64, 0xffff, 0x4000, 0x8000, 0x0, 0x0, 0x0},
remote_settings = {0x1000, 0x0, 0x64, 0x200000, 0x4000, 0x7fffffff, 0x0, 0x0, 0x0},
{rxf_len, rxf_type, rxf_flags, rxf_stream} = {4, 3, 0x0, 57},
}
client = 2a02:a18:9159:9c01:2906:de9e:9c8a:519d 55668 /var/run/varnish.sock,
local.endpoint = /var/run/varnish.sock,
local.socket = a2,
local.ip = 0.0.0.0:0,
remote.ip = 0.0.0.0:0,
server.ip = 2a02:2350:6::b788:3355:443,
client.ip = 2a02:a18:9159:9c01:2906:de9e:9c8a:519d:55668,
},
ws = 0x734199845538 {
id = \"req\",
{s, f, r, e} = {0x73419984b140, +0, +107128, +107128},
},
http_conn = 0x73419984b0b8 {
fd = 216 (@0x727f88caea44),
doclose = null(Not Closing)
ws = 0x734199845538 {
[Already dumped, see above]
},
{rxbuf_b, rxbuf_e} = {0x73419984b140, 0x73419984b140},
{pipeline_b, pipeline_e} = {(nil), (nil)},
content_length = 0,
body_status = NULL,
first_byte_timeout = 0.000000,
between_bytes_timeout = 0.000000,
},
http[req] = 0x7341998455e8 {
ws = NULL
hdrs {
},
},
vdc = 0x73419984b058 {
.magic = 0x00000000 EXPECTED: VDP_CTX_MAGIC=0xee501df7
}
vcl[vcl] = NULL
flags = {
},
privs = 0x7341998455d8 {
},
top = 0x73419984b120 {
req = 0x7341998453e0 {
[Already dumped, see above]
},
privs = 0x73419984b138 {
},
vcl[vcl0] = NULL
},
},
thr.busyobj = NULL
thr.worker = 0x734bb316b320 {
ws = 0x734bb316b3a0 {
id = \"wrk\",
{s, f, r, e} = {0x734bb316a880, +0, (nil), +2040},
},
VCL::method = BACKEND_RESPONSE,
VCL::methods = {},
},
vmods = {
std = {p=0x734bb2a2c500, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"43fbfd6051f0ea3760d04b893ebebcb5351ad263\", version=\"Varnish trunk\"},
one = {p=0x734bb2a2c580, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"5a495ac72e70c4821f77fe99804209a90ea35ae6\", version=\"libvmod-one 0.3\"},
maxminddb = {p=0x734bb2a2c680, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"8d3077d81ff50a9686978d0d72e06654967fdee1\", version=\"libvmod-maxminddb 0.3\"},
directors = {p=0x734bb2a2c700, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"43fbfd6051f0ea3760d04b893ebebcb5351ad263\", version=\"Varnish trunk\"},
xkey = {p=0x734bb2a2c780, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"ee73bb6cb5d7419dc71daa4baaae076bf6bbfb4b\", version=\"varnish-modules 0.26.0\"},
header = {p=0x734bb2a2c800, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"ee73bb6cb5d7419dc71daa4baaae076bf6bbfb4b\", version=\"varnish-modules 0.26.0\"},
cookie = {p=0x734bb2a2c900, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"43fbfd6051f0ea3760d04b893ebebcb5351ad263\", version=\"Varnish trunk\"},
vsthrottle = {p=0x734bb2a2c980, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"ee73bb6cb5d7419dc71daa4baaae076bf6bbfb4b\", version=\"varnish-modules 0.26.0\"},
bodyaccess = {p=0x734bb2a2ca00, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"ee73bb6cb5d7419dc71daa4baaae076bf6bbfb4b\", version=\"varnish-modules 0.26.0\"},
blob = {p=0x734bb2a2ca80, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"43fbfd6051f0ea3760d04b893ebebcb5351ad263\", version=\"Varnish trunk\"},
digest = {p=0x734bb2a2cb80, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=0.0,
vcs=\"bf26880ece69f2cd321cbf57dd59b687c24920e3\", version=\"libvmod-digest 1.0.3\"},
xcounter = {p=0x734bb2a2cc00, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"6d9d8d3ac261a49b11453cdd37da5594619ca335\", version=\"libvmod-xcounter 0.1\"},
re = {p=0x734bb2a2cc80, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"79313a2e71b6a2f478dfa57c821697a2f3da00cc\", version=\"libvmod-re 2.6.0\"},
taskvar = {p=0x734bb2a2cd80, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"ce0f348f87f63aca9b2908265d4b4028830d9530\", version=\"varnish-objvar 0.1\"},
topvar = {p=0x734bb2a2ce00, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"ce0f348f87f63aca9b2908265d4b4028830d9530\", version=\"varnish-objvar 0.1\"},
querystring = {p=0x734bb2a2cf00, abi=\"Varnish trunk 43fbfd6051f0ea3760d04b893ebebcb5351ad263\", vrt=21.0,
vcs=\"NOGIT\", version=\"libvmod-querystring 2.0.3\"},
},
pools = {
pool = 0x734b90400140 {
nidle = 396,
nthr = 1202,
lqueue = 0
},
pool = 0x734b90400000 {
nidle = 286,
nthr = 1130,
lqueue = 0
},
},