- doris-2.1.6
- hive-exec:2.3.5 provided
调用JavaUDF后发现资源未完全释放(内存升高,查询完成后内存不降低,生成的临时文件未正确删除),经定位,是调用JavaUDF的查询存在进程不退出的情况。
编写一个UDF使用java.io.File.deleteOnExit
简单测试,代码如下
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import java.io.File;
import java.io.IOException;
@Description(name = "exit_test", value = "_FUNC_(string) - 家目录创建一个tmp文件,进程结束自动删除")
@SuppressWarnings({"unused"})
public class ExitTest extends UDF {
public String evaluate(String prefix) {
String directory = System.getProperty("user.home");
boolean ret = true;
File file = new File(directory);
if(file.isDirectory()){
try {
File tempFile = File.createTempFile(prefix, ".tmp", file);
tempFile.deleteOnExit();
} catch (IOException e) {
ret = false;
}
}
return prefix + ": " + ret;
}
}
use demo;
drop function exit_test(text);
create function exit_test(text) return text properties (
"file" = "file:///opt/doris/udf/exit_test.jar",
"symbol" = "ExitTest",
"always_nullable" = "true",
"type" = "JAVA_UDF"
);
select exit_test("write_test_delete_on_exit");
调用后发现,执行查询的BE节点会在家目录生成write_test_delete_on_exit*.tmp
,查询结束文件未删除,断开与FE的连接(退出会话),文件也未删除。