From 285b73d624180763a91fc8abb694c0a39d55004b Mon Sep 17 00:00:00 2001 From: lylylyly2 <46030243+lylylyly2@users.noreply.github.com> Date: Tue, 5 Jan 2021 01:31:37 +0800 Subject: [PATCH] #1612 support download large file. (#1642) --- .../term/impl/http/DirectoryBrowser.java | 149 ++++++++++++++++-- .../term/impl/http/HttpRequestHandler.java | 51 +++--- 2 files changed, 165 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/DirectoryBrowser.java b/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/DirectoryBrowser.java index 216cd8717..6fc6cece6 100644 --- a/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/DirectoryBrowser.java +++ b/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/DirectoryBrowser.java @@ -3,17 +3,40 @@ package com.taobao.arthas.core.shell.term.impl.http; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.Calendar; import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Locale; +import java.util.TimeZone; +import com.alibaba.arthas.deps.org.slf4j.Logger; +import com.alibaba.arthas.deps.org.slf4j.LoggerFactory; import com.taobao.arthas.common.IOUtils; - +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelProgressiveFuture; +import io.netty.channel.ChannelProgressiveFutureListener; +import io.netty.channel.DefaultFileRegion; import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpChunkedInput; import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; +import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.ssl.SslHandler; +import io.netty.handler.stream.ChunkedFile; + +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; /** * @@ -22,6 +45,10 @@ import io.netty.handler.codec.http.HttpVersion; */ public class DirectoryBrowser { + public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz"; + public static final String HTTP_DATE_GMT_TIMEZONE = "GMT"; + public static final long MIN_NETTY_DIRECT_SEND_SIZE = 10 * 1024 * 1024; + private static final Logger logger = LoggerFactory.getLogger(DirectoryBrowser.class); //@formatter:off private static String pageHeader = "\n" + "\n" + @@ -108,14 +135,24 @@ public class DirectoryBrowser { return sb.toString(); } - public static DefaultFullHttpResponse view(File dir, String path, HttpVersion version) throws IOException { + /** + * write data here,still return not null just to know succeeded. + * @param dir + * @param path + * @param request + * @param ctx + * @return + * @throws IOException + */ + public static DefaultFullHttpResponse directView(File dir, String path, FullHttpRequest request, ChannelHandlerContext ctx) throws IOException { if (path.startsWith("/")) { path = path.substring(1, path.length()); } + // path maybe: arthas-output/20201225-203454.svg // 需要取 dir的parent来去掉前缀 File file = new File(dir.getParent(), path); - + HttpVersion version = request.protocolVersion(); if (isSubFile(dir, file)) { DefaultFullHttpResponse fullResp = new DefaultFullHttpResponse(version, HttpResponseStatus.OK); @@ -127,22 +164,112 @@ public class DirectoryBrowser { String renderResult = renderDir(file); fullResp.content().writeBytes(renderResult.getBytes("utf-8")); fullResp.headers().set(HttpHeaderNames.CONTENT_TYPE, "text/html; charset=utf-8"); + ctx.write(fullResp); + ChannelFuture future = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); + future.addListener(ChannelFutureListener.CLOSE); + return fullResp; } else { - FileInputStream fileInputStream = new FileInputStream(file); - try { - byte[] content = IOUtils.getBytes(fileInputStream); - fullResp.content().writeBytes(content); - HttpUtil.setContentLength(fullResp, fullResp.content().readableBytes()); - } finally { - IOUtils.close(fileInputStream); + logger.info("get file now. file:" + file.getPath()); + if (file.isHidden() || !file.exists() || file.isDirectory() || !file.isFile()) { + return null; } + + RandomAccessFile raf; + try { + raf = new RandomAccessFile(file, "r"); + } catch (Exception ignore) { + return null; + } + long fileLength = raf.length(); + if (fileLength < MIN_NETTY_DIRECT_SEND_SIZE){ + FileInputStream fileInputStream = new FileInputStream(file); + try { + byte[] content = IOUtils.getBytes(fileInputStream); + fullResp.content().writeBytes(content); + HttpUtil.setContentLength(fullResp, fullResp.content().readableBytes()); + } finally { + IOUtils.close(fileInputStream); + } + ctx.write(fullResp); + ChannelFuture future = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); + future.addListener(ChannelFutureListener.CLOSE); + return fullResp; + } + logger.info("file {} size bigger than 10MB, send by future.",file.getName()); + HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); + HttpUtil.setContentLength(response, fileLength); + setContentTypeHeader(response, file); + setDateAndCacheHeaders(response, file); + if (HttpUtil.isKeepAlive(request)) { + response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE); + } + + // Write the initial line and the header. + ctx.write(response); + // Write the content. + ChannelFuture sendFileFuture; + ChannelFuture lastContentFuture; + if (ctx.pipeline().get(SslHandler.class) == null) { + sendFileFuture = + ctx.write(new DefaultFileRegion(raf.getChannel(), 0, fileLength), ctx.newProgressivePromise()); + // Write the end marker. + lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); + } else { + sendFileFuture = + ctx.writeAndFlush(new HttpChunkedInput(new ChunkedFile(raf, 0, fileLength, 8192)), + ctx.newProgressivePromise()); + // HttpChunkedInput will write the end marker (LastHttpContent) for us. + lastContentFuture = sendFileFuture; + } + + sendFileFuture.addListener(new ChannelProgressiveFutureListener() { + @Override + public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) { + if (total < 0) { // total unknown + logger.info(future.channel() + " Transfer progress: " + progress); + } else { + logger.info(future.channel() + " Transfer progress: " + progress + " / " + total); + } + } + + @Override + public void operationComplete(ChannelProgressiveFuture future) { + logger.info(future.channel() + " Transfer complete."); + } + }); + + // Decide whether to close the connection or not. + if (!HttpUtil.isKeepAlive(request)) { + // Close the connection when the whole content is written out. + lastContentFuture.addListener(ChannelFutureListener.CLOSE); + } + return fullResp; } - return fullResp; } return null; } + private static void setDateAndCacheHeaders(HttpResponse response, File fileToCache) { + SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US); + dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE)); + // Date header + Calendar time = new GregorianCalendar(); + response.headers().set(HttpHeaderNames.DATE, dateFormatter.format(time.getTime())); + + // Add cache headers + time.add(Calendar.SECOND, 3600); + response.headers().set(HttpHeaderNames.EXPIRES, dateFormatter.format(time.getTime())); + response.headers().set(HttpHeaderNames.CACHE_CONTROL, "private, max-age=" + 3600); + response.headers().set( + HttpHeaderNames.LAST_MODIFIED, dateFormatter.format(new Date(fileToCache.lastModified()))); + } + + private static void setContentTypeHeader(HttpResponse response, File file) { + String contentType = "application/octet-stream"; + // 暂时hardcode 大文件的content-type + response.headers().set(HttpHeaderNames.CONTENT_TYPE, contentType); + } public static boolean isSubFile(File parent, File child) throws IOException { String parentPath = parent.getCanonicalPath(); String childPath = child.getCanonicalPath(); diff --git a/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/HttpRequestHandler.java b/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/HttpRequestHandler.java index 4d683de63..ba3777855 100644 --- a/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/HttpRequestHandler.java +++ b/core/src/main/java/com/taobao/arthas/core/shell/term/impl/http/HttpRequestHandler.java @@ -1,18 +1,11 @@ package com.taobao.arthas.core.shell.term.impl.http; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URL; - import com.alibaba.arthas.deps.org.slf4j.Logger; import com.alibaba.arthas.deps.org.slf4j.LoggerFactory; import com.taobao.arthas.common.IOUtils; import com.taobao.arthas.core.server.ArthasBootstrap; import com.taobao.arthas.core.shell.term.impl.http.api.HttpApiHandler; import com.taobao.arthas.core.shell.term.impl.httptelnet.HttpTelnetTermServer; - import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; @@ -24,13 +17,19 @@ import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.LastHttpContent; import io.termd.core.http.HttpTtyConnection; import io.termd.core.util.Logging; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; + import static com.taobao.arthas.core.util.HttpUtils.createRedirectResponse; import static com.taobao.arthas.core.util.HttpUtils.createResponse; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; /** * @author Julien Viet @@ -46,6 +45,7 @@ public class HttpRequestHandler extends SimpleChannelInboundHandler