【TDengine 使用环境】生产环境
【TDengine 版本】TDengine TSDB-OSS ver:3.3.8.8
【操作系统以及版本】Centos7.9
【部署方式】非容器
【集群节点数】3
【集群副本数】3
【描述业务影响】文件描述符删除未释放,导致磁盘空间不断扩大
【问题复现路径/shan】做过哪些操作出现的问题
以下是几个流任务示例
示例1
create stream if not exists auc_sw_service.stream_auc_service_provider interval(1m) sliding(1m) from auc_sw_service.sw_service_provider partition by tbname, domain, `spanRole` stream_options(watermark(30s) | fill_history) into auc_sw_service_min.sw_service_provider_ip output_subtable(concat(tbname, '_ip')) (_ts, `serviceInstance` composite key) tags ( domain nchar (64) as domain, spanrole nchar (16) as `spanRole`) as select _twstart as _ts, `serviceInstance` from %%tbname where _c0 >= _twstart and _c0 <= _twend partition by domain, `spanRole`, `serviceInstance`;
示例2
create stream if not exists grayproduct_sw_service.stream_grayproduct_service_neomq_consumer interval(1m) sliding(1m) from grayproduct_sw_service.sw_service_neomq_consumer partition by tbname, domain, `spanRole` stream_options(watermark(30s) | fill_history) into grayproduct_sw_service_min.sw_service_neomq_consumer_ip output_subtable(concat(tbname, '_ip')) (_ts, `serviceInstance` composite key) tags ( domain nchar (64) as domain, spanrole nchar (16) as `spanRole`) as select _twstart as _ts, `serviceInstance` from %%tbname where _c0 >= _twstart and _c0 <= _twend partition by domain, `spanRole`, `serviceInstance`;
示例3
create stream if not exists p05_sw_service.stream_p05_service_provider interval(1m) sliding(1m) from p05_sw_service.sw_service_provider partition by tbname, domain, `spanRole` stream_options(watermark(30s) | fill_history) into p05_sw_service_min.sw_service_provider_ip output_subtable(concat(tbname, '_ip')) (_ts, `serviceInstance` composite key) tags ( domain nchar (64) as domain, spanrole nchar (16) as `spanRole`) as select _twstart as _ts, `serviceInstance` from %%tbname where _c0 >= _twstart and _c0 <= _twend partition by domain, `spanRole`, `serviceInstance`;
【遇到的问题:问题现象及影响】
以下是未释放文件描述符查看
[root@tdengine-97 tmp]# lsof +L1 | grep '/opt/' | grep deleted
taosd 24710 root 1065u REG 253,16 1650081900 0 5242913 /opt/tmp/tdengine-paged-buf-24710-29 (deleted)
taosd 24710 root 1079u REG 253,16 918527100 0 5242896 /opt/tmp/tdengine-paged-buf-24710-12 (deleted)
taosd 24710 root 1100u REG 253,16 15472990000 0 5242890 /opt/tmp/tdengine-paged-buf-24710-6 (deleted)
taosd 24710 root 1275u REG 253,16 691116500 0 5242911 /opt/tmp/tdengine-paged-buf-24710-27 (deleted)
taosd 24710 root 1353u REG 253,16 32712661800 0 5242885 /opt/tmp/tdengine-paged-buf-24710-1 (deleted)
taosd 24710 root 1366u REG 253,16 1069144700 0 5242895 /opt/tmp/tdengine-paged-buf-24710-11 (deleted)
taosd 24710 root 1465u REG 253,16 33512042900 0 5242888 /opt/tmp/tdengine-paged-buf-24710-4 (deleted)
taosd 24710 root 1495u REG 253,16 948309500 0 5242886 /opt/tmp/tdengine-paged-buf-24710-2 (deleted)
taosd 24710 root 1502u REG 253,16 1605740400 0 5242901 /opt/tmp/tdengine-paged-buf-24710-17 (deleted)
taosd 24710 root 1600u REG 253,16 1613145000 0 5242902 /opt/tmp/tdengine-paged-buf-24710-18 (deleted)
taosd 24710 root 1620u REG 253,16 893918900 0 5242897 /opt/tmp/tdengine-paged-buf-24710-13 (deleted)
taosd 24710 root 1748u REG 253,16 1932530900 0 5242903 /opt/tmp/tdengine-paged-buf-24710-19 (deleted)
taosd 24710 root 1755u REG 253,16 82061500 0 5242887 /opt/tmp/tdengine-paged-buf-24710-3 (deleted)
taosd 24710 root 1815u REG 253,16 1011535600 0 5242894 /opt/tmp/tdengine-paged-buf-24710-10 (deleted)
taosd 24710 root 1816u REG 253,16 1090468800 0 5242908 /opt/tmp/tdengine-paged-buf-24710-24 (deleted)
taosd 24710 root 1844u REG 253,16 48267311500 0 5242889 /opt/tmp/tdengine-paged-buf-24710-5 (deleted)
taosd 24710 root 1858u REG 253,16 824981500 0 5242912 /opt/tmp/tdengine-paged-buf-24710-28 (deleted)
taosd 24710 root 1897u REG 253,16 1084421300 0 5242891 /opt/tmp/tdengine-paged-buf-24710-7 (deleted)
taosd 24710 root 1899u REG 253,16 1469505600 0 5242893 /opt/tmp/tdengine-paged-buf-24710-9 (deleted)
taosd 24710 root 1924u REG 253,16 993938400 0 5242898 /opt/tmp/tdengine-paged-buf-24710-14 (deleted)
taosd 24710 root 1944u REG 253,16 918227800 0 5242892 /opt/tmp/tdengine-paged-buf-24710-8 (deleted)
taosd 24710 root 1973u REG 253,16 20315500 0 5242915 /opt/tmp/tdengine-paged-buf-24710-31 (deleted)
taosd 24710 root 2002u REG 253,16 1262640100 0 5242899 /opt/tmp/tdengine-paged-buf-24710-15 (deleted)
taosd 24710 root 2060u REG 253,16 1590508900 0 5242909 /opt/tmp/tdengine-paged-buf-24710-25 (deleted)
taosd 24710 root 2082u REG 253,16 784399700 0 5242914 /opt/tmp/tdengine-paged-buf-24710-30 (deleted)
taosd 24710 root 2091u REG 253,16 1686276700 0 5242906 /opt/tmp/tdengine-paged-buf-24710-22 (deleted)
taosd 24710 root 2135u REG 253,16 1688863800 0 5242905 /opt/tmp/tdengine-paged-buf-24710-21 (deleted)
taosd 24710 root 2154u REG 253,16 1938266800 0 5242910 /opt/tmp/tdengine-paged-buf-24710-26 (deleted)
taosd 24710 root 2175u REG 253,16 1796374000 0 5242904 /opt/tmp/tdengine-paged-buf-24710-20 (deleted)
taosd 24710 root 2202u REG 253,16 1802692100 0 5242900 /opt/tmp/tdengine-paged-buf-24710-16 (deleted)
taosd 24710 root 2203u REG 253,16 1496959200 0 5242907 /opt/tmp/tdengine-paged-buf-24710-23 (deleted)
taosd 24710 root 2295u REG 253,16 8446000 0 5242916 /opt/tmp/tdengine-paged-buf-24710-32 (deleted)
即使将文件描述符截断或者使用 > 追加为空,在使用一段时间后该文件还是会写入,空间还是会被占用,以下是临时释放空间命令
# 截断
truncate -s 0 /proc/24710/fd/1065
# 追加
> /proc/24710/fd/1065
【资源配置】16c32g,200G运行磁盘,3T数据磁盘
【报错完整截图】(不要大段的粘贴报错代码,论坛直接看报错代码不直观)