Nicksxs's Blog

What hurts more, the pain of hard work or the pain of regret?

我们在使用 Java 的日志库的时候,比如我们现在项目在用的 logback,可以配置滚动策略,简单介绍下启动逻辑,这里我们定义的是
ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy 它是继承了 ch.qos.logback.core.rolling.TimeBasedRollingPolicy
它的启动方法就是下面这个 start 方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
public void start() {
SizeAndTimeBasedFNATP<E> sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP<E>(Usage.EMBEDDED);
if(maxFileSize == null) {
addError("maxFileSize property is mandatory.");
return;
} else {
addInfo("Archive files will be limited to ["+maxFileSize+"] each.");
}

sizeAndTimeBasedFNATP.setMaxFileSize(maxFileSize);
timeBasedFileNamingAndTriggeringPolicy = sizeAndTimeBasedFNATP;

if(!isUnboundedTotalSizeCap() && totalSizeCap.getSize() < maxFileSize.getSize()) {
addError("totalSizeCap of ["+totalSizeCap+"] is smaller than maxFileSize ["+maxFileSize+"] which is non-sensical");
return;
}

// most work is done by the parent
super.start();
}

配置了 timeBasedFileNamingAndTriggeringPolicy 策略

然后调用了父类的启动方法,主要看下父类的

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
public void start() {
// set the LR for our utility object
renameUtil.setContext(this.context);

// find out period from the filename pattern
if (fileNamePatternStr != null) {
fileNamePattern = new FileNamePattern(fileNamePatternStr, this.context);
determineCompressionMode();
} else {
addWarn(FNP_NOT_SET);
addWarn(CoreConstants.SEE_FNP_NOT_SET);
throw new IllegalStateException(FNP_NOT_SET + CoreConstants.SEE_FNP_NOT_SET);
}

compressor = new Compressor(compressionMode);
compressor.setContext(context);

// wcs : without compression suffix
fileNamePatternWithoutCompSuffix = new FileNamePattern(Compressor.computeFileNameStrWithoutCompSuffix(fileNamePatternStr, compressionMode), this.context);

addInfo("Will use the pattern " + fileNamePatternWithoutCompSuffix + " for the active file");

if (compressionMode == CompressionMode.ZIP) {
String zipEntryFileNamePatternStr = transformFileNamePattern2ZipEntry(fileNamePatternStr);
zipEntryFileNamePattern = new FileNamePattern(zipEntryFileNamePatternStr, context);
}

if (timeBasedFileNamingAndTriggeringPolicy == null) {
timeBasedFileNamingAndTriggeringPolicy = new DefaultTimeBasedFileNamingAndTriggeringPolicy<E>();
}
timeBasedFileNamingAndTriggeringPolicy.setContext(context);
timeBasedFileNamingAndTriggeringPolicy.setTimeBasedRollingPolicy(this);
timeBasedFileNamingAndTriggeringPolicy.start();

if (!timeBasedFileNamingAndTriggeringPolicy.isStarted()) {
addWarn("Subcomponent did not start. TimeBasedRollingPolicy will not start.");
return;
}

// the maxHistory property is given to TimeBasedRollingPolicy instead of to
// the TimeBasedFileNamingAndTriggeringPolicy. This makes it more convenient
// for the user at the cost of inconsistency here.
if (maxHistory != UNBOUND_HISTORY) {
archiveRemover = timeBasedFileNamingAndTriggeringPolicy.getArchiveRemover();
archiveRemover.setMaxHistory(maxHistory);
archiveRemover.setTotalSizeCap(totalSizeCap.getSize());
if (cleanHistoryOnStart) {
addInfo("Cleaning on start up");
Date now = new Date(timeBasedFileNamingAndTriggeringPolicy.getCurrentTime());
cleanUpFuture = archiveRemover.cleanAsynchronously(now);
}
} else if (!isUnboundedTotalSizeCap()) {
addWarn("'maxHistory' is not set, ignoring 'totalSizeCap' option with value ["+totalSizeCap+"]");
}

super.start();
}
  • 第一步是给 renameUtil 设置 context

  • 第二步是判断 fileNamePatternStr是否已配置,没配置会报错,如果配置了就会去判断压缩格式,是否要压缩以及压缩的是 gz 还是 zip

  • 第三步就是设置压缩器了

  • 第四步是判断文件后缀格式,注意是不带压缩格式的

  • 第五步是判断 timeBasedFileNamingAndTriggeringPolicy 是否已设置,这里是在子类里已经设置了 ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP 否则就是 ch.qos.logback.core.rolling.DefaultTimeBasedFileNamingAndTriggeringPolicy 这个默认的

  • 第六步比较重要就是启动 timeBasedFileNamingAndTriggeringPolicy

timeBasedFileNamingAndTriggeringPolicy 的启动逻辑里首先是调用了 SizeAndTimeBasedFNATP 的 start 方法,然后里面最开始调用了父类的 start

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
public void start() {
// we depend on certain fields having been initialized in super class
super.start();

if(usage == Usage.DIRECT) {
addWarn(CoreConstants.SIZE_AND_TIME_BASED_FNATP_IS_DEPRECATED);
addWarn("For more information see "+MANUAL_URL_PREFIX+"appenders.html#SizeAndTimeBasedRollingPolicy");
}

if (!super.isErrorFree())
return;


if (maxFileSize == null) {
addError("maxFileSize property is mandatory.");
withErrors();
}

if (!validateDateAndIntegerTokens()) {
withErrors();
return;
}

archiveRemover = createArchiveRemover();
archiveRemover.setContext(context);

// we need to get the correct value of currentPeriodsCounter.
// usually the value is 0, unless the appender or the application
// is stopped and restarted within the same period
String regex = tbrp.fileNamePattern.toRegexForFixedDate(dateInCurrentPeriod);
String stemRegex = FileFilterUtil.afterLastSlash(regex);

computeCurrentPeriodsHighestCounterValue(stemRegex);

if (isErrorFree()) {
started = true;
}
}

也就是下面的代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
public void start() {
DateTokenConverter<Object> dtc = tbrp.fileNamePattern.getPrimaryDateTokenConverter();
if (dtc == null) {
throw new IllegalStateException("FileNamePattern [" + tbrp.fileNamePattern.getPattern() + "] does not contain a valid DateToken");
}

if (dtc.getTimeZone() != null) {
rc = new RollingCalendar(dtc.getDatePattern(), dtc.getTimeZone(), Locale.getDefault());
} else {
rc = new RollingCalendar(dtc.getDatePattern());
}
addInfo("The date pattern is '" + dtc.getDatePattern() + "' from file name pattern '" + tbrp.fileNamePattern.getPattern() + "'.");
rc.printPeriodicity(this);

if (!rc.isCollisionFree()) {
addError("The date format in FileNamePattern will result in collisions in the names of archived log files.");
addError(CoreConstants.MORE_INFO_PREFIX + COLLIDING_DATE_FORMAT_URL);
withErrors();
return;
}

setDateInCurrentPeriod(new Date(getCurrentTime()));
if (tbrp.getParentsRawFileProperty() != null) {
File currentFile = new File(tbrp.getParentsRawFileProperty());
if (currentFile.exists() && currentFile.canRead()) {
setDateInCurrentPeriod(new Date(currentFile.lastModified()));
}
}
addInfo("Setting initial period to " + dateInCurrentPeriod);
computeNextCheck();
}

前面是日期时区等配置处理,然后是判断按日期生成的文件会不会冲突,接下去是设置当前时间段,如果配置了日志文件的话就会把当前时间段设置成已有的日志文件的最后更改时间,最后的 computeNextCheck 比较重要

主要是下面的方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
protected void computeNextCheck() {
nextCheck = rc.getNextTriggeringDate(dateInCurrentPeriod).getTime();
}
public Date getNextTriggeringDate(Date now) {
return getEndOfNextNthPeriod(now, 1);
}
public Date getEndOfNextNthPeriod(Date now, int periods) {
return innerGetEndOfNextNthPeriod(this, this.periodicityType, now, periods);
}
static private Date innerGetEndOfNextNthPeriod(Calendar cal, PeriodicityType periodicityType, Date now, int numPeriods) {
cal.setTime(now);
switch (periodicityType) {
case TOP_OF_MILLISECOND:
cal.add(Calendar.MILLISECOND, numPeriods);
break;

case TOP_OF_SECOND:
cal.set(Calendar.MILLISECOND, 0);
cal.add(Calendar.SECOND, numPeriods);
break;

case TOP_OF_MINUTE:
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.add(Calendar.MINUTE, numPeriods);
break;

case TOP_OF_HOUR:
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.add(Calendar.HOUR_OF_DAY, numPeriods);
break;

case TOP_OF_DAY:
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.add(Calendar.DATE, numPeriods);
break;

case TOP_OF_WEEK:
cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.add(Calendar.WEEK_OF_YEAR, numPeriods);
break;

case TOP_OF_MONTH:
cal.set(Calendar.DATE, 1);
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.add(Calendar.MONTH, numPeriods);
break;

default:
throw new IllegalStateException("Unknown periodicity type.");
}

return cal.getTime();

 会按照我们设置的 FileNamePattern 中的 datePattern 来推断 periodicityType,比如我们是小时滚动的,那就是 TOP_OF_HOUR ,算出来下一个时间检查点,后面会按这个判断是否作为触发事件触发日志滚动更新

父类的 start 逻辑讲完以后,子类的其实比较简单,先判断使用方式,我们这是嵌入式的,然后是父类有没有产生错误,继续是最大文件大小是否设置了,再判断日期格式是否正常,然后是归档移除类的创建,并设置到上下文中,然后计算当前时间的最大日志文件计数器,最后判断是否报错,没有的话就启动成功了

php 代码调试里的神器就是 echo 111;exit; 但是对于使用了接口和继承比较多的话,有时候比较难找,可能定位到了一段代码但是不知道怎么调用过来的,这时候就可以用这个方法

总结下来有三种,

第一种是最巧妙的

1
2
3
function a() {
echo 111;exit;
}

比如本来是上面这样子,那么其实我们可以主动new 个异常

1
2
3
4
5
function a() {
$e = new Exception();
print_r($e->getTraceAsString());
echo 111;exit;
}

这样我的 trace 调用链路就出来了

第二种

这个就是比较简单的,调用 php 自身提供的方法

1
debug_backtrace(int $options = DEBUG_BACKTRACE_PROVIDE_OBJECT, int $limit = 0): array

第一个参数是个掩码

debug_backtrace()Populates both indexes
debug_backtrace(DEBUG_BACKTRACE_PROVIDE_OBJECT)
debug_backtrace(1)
debug_backtrace(0)Omits index "object" and populates index "args".
debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS)Omits index "object" and index "args".
debug_backtrace(2)
debug_backtrace(DEBUG_BACKTRACE_PROVIDE_OBJECTDEBUG_BACKTRACE_IGNORE_ARGS)
debug_backtrace(3)

第二个参数是限制栈深度

第三种

这个也是用自身的方法

1
debug_print_backtrace(int $options = 0, int $limit = 0): void

这里的第一个参数只有一个可以传的

DEBUG_BACKTRACE_IGNORE_ARGSWhether or not to omit the “args” index, and thus all the function/method arguments, to save memory.

就是隐藏参数,不然如果对于一些框架代码,这个打印会非常大,需要注意下

昨天同学问我是不是数据库主从延迟有点高,可能有一分多钟,然后我就去看了rds 的监控,发现主实例上的监控显示的延迟才 1.2 秒,而且是最高 1.2 秒,感觉这样的话应该就没啥问题,然后同学跟我说他加了日志,大致的逻辑是主库数据落库以后就会发一条 mq 消息出来,然后消费者接收到以后回去从库查一下这个数据,结果发现延迟了 90 多秒才查到数据,这种情况比较可能的猜测是阿里云这个监控的逻辑可能是从库在获得第一条同步数据的时候,而不是最终同步完成,但是跟阿里云咨询了并不是,使用的就是 show slave status 结果里的 Seconds_Behind_Master 指标,那这第一种情况就否定掉了,这里其实是犯了个错误,应该去从库看这个延迟的,不过对于阿里云来说在 rds 监控是看不到从库的监控的,只能到 das,也就是阿里云的数据库自治服务可以看到,这里能看到从库的延迟监控,发现的确有这么高,这样就要考虑为啥会出现这种情况,阿里云同学反馈的是这段时间的 iops 很高,并且 cpu 也比较高,让我排查下 binlog,这里就碰到一个小问题,阿里云 rds 的线上实例我们没法在本地连接,并且密码也是在代码里加密过的,去服务器上连接一方面需要装 mysql 客户端,另一方面是怕拉取日志会有性能影响,幸好阿里云这点做的比较好,在 rds 的”备份恢复”–> “日志备份”菜单里可以找到binlog 文件, 在这里其实大致就发现了问题,因为出问题的时间段内升成 binlog 的量大大超过其他时间段,然后通过内网下载后就可以对 binlog 进行分析了,这里我们用到了 mysqlbinlog 工具,主要就是找具体是哪些写入导致这个问题,mysqlbinlog 可以在 mysql 官网下载 mysql 的压缩包,注意是压缩包,这样解压了直接用就好了,不用完整安装 mysql,一般我们需要对 binlog 进行 base64 的反编码,

1
./mysqlbinlog -vv --base64-output=decode-rows mysql-bin.xxxx | less

这样查看里面的具体信息,发现是大量的 insert 数据,再经过排查发现同一个 binlog 文件近 500M 全是同一个表的数据插入,再根据表对应的业务查找发现是有个业务逻辑会在这个时间点全量删除后在生成插入数据,后续需要进行优化

最近同学在把 springboot 升级到 2.x 版本的过程中碰到了小问题,可能升级变更里能找到信息,不过我们以学习为目的,可以看看代码是怎么样的
报错是在这段代码里的
org.apache.tomcat.util.http.fileupload.util.LimitedInputStream#checkLimit

1
2
3
4
5
private void checkLimit() throws IOException {
if (count > sizeMax) {
raiseError(sizeMax, count);
}
}

其中的 raiseError 是个抽象方法

1
2
protected abstract void raiseError(long pSizeMax, long pCount)
throws IOException;

具体的实现是在

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
public FileItemStreamImpl(FileItemIteratorImpl pFileItemIterator, String pName, String pFieldName, String pContentType, boolean pFormField, long pContentLength) throws FileUploadException, IOException {
this.fileItemIteratorImpl = pFileItemIterator;
this.name = pName;
this.fieldName = pFieldName;
this.contentType = pContentType;
this.formField = pFormField;
long fileSizeMax = this.fileItemIteratorImpl.getFileSizeMax();
if (fileSizeMax != -1L && pContentLength != -1L && pContentLength > fileSizeMax) {
FileSizeLimitExceededException e = new FileSizeLimitExceededException(String.format("The field %s exceeds its maximum permitted size of %s bytes.", this.fieldName, fileSizeMax), pContentLength, fileSizeMax);
e.setFileName(pName);
e.setFieldName(pFieldName);
throw new FileUploadIOException(e);
} else {
final MultipartStream.ItemInputStream itemStream = this.fileItemIteratorImpl.getMultiPartStream().newInputStream();
InputStream istream = itemStream;
if (fileSizeMax != -1L) {
istream = new LimitedInputStream(itemStream, fileSizeMax) {
protected void raiseError(long pSizeMax, long pCount) throws IOException {
itemStream.close(true);
FileSizeLimitExceededException e = new FileSizeLimitExceededException(String.format("The field %s exceeds its maximum permitted size of %s bytes.", FileItemStreamImpl.this.fieldName, pSizeMax), pCount, pSizeMax);
e.setFieldName(FileItemStreamImpl.this.fieldName);
e.setFileName(FileItemStreamImpl.this.name);
throw new FileUploadIOException(e);
}
};
}

this.stream = (InputStream)istream;
}
}

后面也会介绍到,这里我们其实主要是要找到这个 pSizeMax 是哪里来的
通过阅读代码会发现跟这个类 MultipartConfigElement 有关系
而在升级后的 springboot 中这个类已经有了自动装配类,也就是
org.springframework.boot.autoconfigure.web.servlet.MultipartAutoConfiguration

有了这个自动装配

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
@Configuration(
proxyBeanMethods = false
)
@ConditionalOnClass({Servlet.class, StandardServletMultipartResolver.class, MultipartConfigElement.class})
@ConditionalOnProperty(
prefix = "spring.servlet.multipart",
name = {"enabled"},
matchIfMissing = true
)
@ConditionalOnWebApplication(
type = Type.SERVLET
)
@EnableConfigurationProperties({MultipartProperties.class})
public class MultipartAutoConfiguration {
private final MultipartProperties multipartProperties;

public MultipartAutoConfiguration(MultipartProperties multipartProperties) {
this.multipartProperties = multipartProperties;
}

@Bean
@ConditionalOnMissingBean({MultipartConfigElement.class, CommonsMultipartResolver.class})
public MultipartConfigElement multipartConfigElement() {
return this.multipartProperties.createMultipartConfig();
}

而这个 MultipartProperties 类中

1
2
3
4
5
6
7
8
9
10
11
@ConfigurationProperties(
prefix = "spring.servlet.multipart",
ignoreUnknownFields = false
)
public class MultipartProperties {
private boolean enabled = true;
private String location;
private DataSize maxFileSize = DataSize.ofMegabytes(1L);
private DataSize maxRequestSize = DataSize.ofMegabytes(10L);
private DataSize fileSizeThreshold = DataSize.ofBytes(0L);
private boolean resolveLazily = false;

并且在前面 createMultipartConfig 中就使用了这个maxFileSize 的默认值

1
2
3
4
5
6
7
8
9
public MultipartConfigElement createMultipartConfig() {
MultipartConfigFactory factory = new MultipartConfigFactory();
PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull();
map.from(this.fileSizeThreshold).to(factory::setFileSizeThreshold);
map.from(this.location).whenHasText().to(factory::setLocation);
map.from(this.maxRequestSize).to(factory::setMaxRequestSize);
map.from(this.maxFileSize).to(factory::setMaxFileSize);
return factory.createMultipartConfig();
}

而在 org.apache.catalina.connector.Request#parseParts 中,会判断 mce 的配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
private void parseParts(boolean explicit) {

// 省略一部分代码

ServletFileUpload upload = new ServletFileUpload();
upload.setFileItemFactory(factory);
upload.setFileSizeMax(mce.getMaxFileSize());
upload.setSizeMax(mce.getMaxRequestSize());

parts = new ArrayList<>();
try {
List<FileItem> items =
upload.parseRequest(new ServletRequestContext(this));
int maxPostSize = getConnector().getMaxPostSize();
int postSize = 0;
Charset charset = getCharset();

主要 org.apache.tomcat.util.http.fileupload.FileUploadBase#parseRequest

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
public List<FileItem> parseRequest(final RequestContext ctx)
throws FileUploadException {
final List<FileItem> items = new ArrayList<>();
boolean successful = false;
try {
final FileItemIterator iter = getItemIterator(ctx);
final FileItemFactory fileItemFactory = Objects.requireNonNull(getFileItemFactory(),
"No FileItemFactory has been set.");
final byte[] buffer = new byte[Streams.DEFAULT_BUFFER_SIZE];
while (iter.hasNext()) {
final FileItemStream item = iter.next();
// Don't use getName() here to prevent an InvalidFileNameException.
final String fileName = item.getName();
final FileItem fileItem = fileItemFactory.createItem(item.getFieldName(), item.getContentType(),
item.isFormField(), fileName);
items.add(fileItem);
try {
Streams.copy(item.openStream(), fileItem.getOutputStream(), true, buffer);
} catch (final FileUploadIOException e) {

其中 org.apache.tomcat.util.http.fileupload.FileUploadBase#getItemIterator

1
2
3
4
5
6
7
8
9
public FileItemIterator getItemIterator(final RequestContext ctx)
throws FileUploadException, IOException {
try {
return new FileItemIteratorImpl(this, ctx);
} catch (final FileUploadIOException e) {
// unwrap encapsulated SizeException
throw (FileUploadException) e.getCause();
}
}

这里就创建了 org.apache.tomcat.util.http.fileupload.impl.FileItemIteratorImpl

1
2
3
4
5
6
7
8
9
10
public FileItemIteratorImpl(final FileUploadBase fileUploadBase, final RequestContext requestContext)
throws FileUploadException, IOException {
this.fileUploadBase = fileUploadBase;
sizeMax = fileUploadBase.getSizeMax();
fileSizeMax = fileUploadBase.getFileSizeMax();
ctx = Objects.requireNonNull(requestContext, "requestContext");
skipPreamble = true;
findNextItem();
}

内部使用了前面给 upload 设置的文件大小上限 upload.setFileSizeMax(mce.getMaxFileSize());

然后在 findNextItem 里执行了初始化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
private boolean findNextItem() throws FileUploadException, IOException {
if (eof) {
return false;
}
if (currentItem != null) {
currentItem.close();
currentItem = null;
}
final MultipartStream multi = getMultiPartStream();
for (;;) {
final boolean nextPart;
if (skipPreamble) {
nextPart = multi.skipPreamble();
} else {
nextPart = multi.readBoundary();
}
if (!nextPart) {
if (currentFieldName == null) {
// Outer multipart terminated -> No more data
eof = true;
return false;
}
// Inner multipart terminated -> Return to parsing the outer
multi.setBoundary(multiPartBoundary);
currentFieldName = null;
continue;
}
final FileItemHeaders headers = fileUploadBase.getParsedHeaders(multi.readHeaders());
if (currentFieldName == null) {
// We're parsing the outer multipart
final String fieldName = fileUploadBase.getFieldName(headers);
if (fieldName != null) {
final String subContentType = headers.getHeader(FileUploadBase.CONTENT_TYPE);
if (subContentType != null
&& subContentType.toLowerCase(Locale.ENGLISH)
.startsWith(FileUploadBase.MULTIPART_MIXED)) {
currentFieldName = fieldName;
// Multiple files associated with this field name
final byte[] subBoundary = fileUploadBase.getBoundary(subContentType);
multi.setBoundary(subBoundary);
skipPreamble = true;
continue;
}
final String fileName = fileUploadBase.getFileName(headers);
currentItem = new FileItemStreamImpl(this, fileName,
fieldName, headers.getHeader(FileUploadBase.CONTENT_TYPE),
fileName == null, getContentLength(headers));
currentItem.setHeaders(headers);
progressNotifier.noteItem();
itemValid = true;
return true;
}
} else {
final String fileName = fileUploadBase.getFileName(headers);
if (fileName != null) {
currentItem = new FileItemStreamImpl(this, fileName,
currentFieldName,
headers.getHeader(FileUploadBase.CONTENT_TYPE),
false, getContentLength(headers));
currentItem.setHeaders(headers);
progressNotifier.noteItem();
itemValid = true;
return true;
}
}
multi.discardBodyData();
}
}

这里面就会 new 这个 FileItemStreamImpl

1
2
3
4
currentItem = new FileItemStreamImpl(this, fileName,
fieldName, headers.getHeader(FileUploadBase.CONTENT_TYPE),
fileName == null, getContentLength(headers));

构造方法比较长

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
public FileItemStreamImpl(final FileItemIteratorImpl pFileItemIterator, final String pName, final String pFieldName,
final String pContentType, final boolean pFormField,
final long pContentLength) throws FileUploadException, IOException {
fileItemIteratorImpl = pFileItemIterator;
name = pName;
fieldName = pFieldName;
contentType = pContentType;
formField = pFormField;
final long fileSizeMax = fileItemIteratorImpl.getFileSizeMax();
if (fileSizeMax != -1 && pContentLength != -1
&& pContentLength > fileSizeMax) {
final FileSizeLimitExceededException e =
new FileSizeLimitExceededException(
String.format("The field %s exceeds its maximum permitted size of %s bytes.",
fieldName, Long.valueOf(fileSizeMax)),
pContentLength, fileSizeMax);
e.setFileName(pName);
e.setFieldName(pFieldName);
throw new FileUploadIOException(e);
}
// OK to construct stream now
final ItemInputStream itemStream = fileItemIteratorImpl.getMultiPartStream().newInputStream();
InputStream istream = itemStream;
if (fileSizeMax != -1) {
istream = new LimitedInputStream(istream, fileSizeMax) {
@Override
protected void raiseError(final long pSizeMax, final long pCount)
throws IOException {
itemStream.close(true);
final FileSizeLimitExceededException e =
new FileSizeLimitExceededException(
String.format("The field %s exceeds its maximum permitted size of %s bytes.",
fieldName, Long.valueOf(pSizeMax)),
pCount, pSizeMax);
e.setFieldName(fieldName);
e.setFileName(name);
throw new FileUploadIOException(e);
}
};
}
stream = istream;
}

fileSizeMax != 0 的时候就会初始化 LimitedInputStream,这就就是会在前面的

org.apache.tomcat.util.http.fileupload.FileUploadBase#parseRequest

1
Streams.copy(item.openStream(), fileItem.getOutputStream(), true, buffer);

这里的 item

1
2
3
4
5
6
final FileItemIterator iter = getItemIterator(ctx);
final FileItemFactory fileItemFactory = Objects.requireNonNull(getFileItemFactory(),
"No FileItemFactory has been set.");
final byte[] buffer = new byte[Streams.DEFAULT_BUFFER_SIZE];
while (iter.hasNext()) {
final FileItemStream item = iter.next();

调用了 FileItemIterator 迭代器的 next

1
2
3
4
5
6
7
8
@Override
public FileItemStream next() throws FileUploadException, IOException {
if (eof || (!itemValid && !hasNext())) {
throw new NoSuchElementException();
}
itemValid = false;
return currentItem;
}

这个 currentItem 就是前面 new 的 FileItemStreamImpl

然后在 Streams.copy 的时候调用 openStream 也就是 org.apache.tomcat.util.http.fileupload.impl.FileItemStreamImpl#openStream

1
2
3
4
5
6
7
@Override
public InputStream openStream() throws IOException {
if (((Closeable) stream).isClosed()) {
throw new FileItemStream.ItemSkippedException();
}
return stream;
}

这里的 stream 就是 FileItemStreamImpl 构造方法最后赋值的 stream,会在大小超过限制时抛出错误

而这个可以通过设置 properties 来修改,spring.servlet.multipart.max-file-size 和 spring.servlet.multipart.max-request-size

1
2
spring.servlet.multipart.max-file-size=100MB
spring.servlet.multipart.max-request-size=100MB

而老版本的 spring.http.multipart.maxFileSize
其实就是配置名称改了下,但是能看一下代码也是有点收获的。

这部分其实之前在讲线程池的时候也有点带到了, 主要是在这个类里
org.apache.catalina.core.ContainerBase.ContainerBackgroundProcessor

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
protected class ContainerBackgroundProcessor implements Runnable {

@Override
public void run() {
processChildren(ContainerBase.this);
}

protected void processChildren(Container container) {
ClassLoader originalClassLoader = null;

try {
if (container instanceof Context) {
Loader loader = ((Context) container).getLoader();
// Loader will be null for FailedContext instances
if (loader == null) {
return;
}

// Ensure background processing for Contexts and Wrappers
// is performed under the web app's class loader
originalClassLoader = ((Context) container).bind(false, null);
}
// 调用 Container 的 backgroundProcess
container.backgroundProcess();
// 然后寻找 children
Container[] children = container.findChildren();
for (Container child : children) {
// 如果 backgroundProcessorDelay <= 0 就调用执行
// 否则代表这个 Container 有之前第八篇说的 StartChild 这种
if (child.getBackgroundProcessorDelay() <= 0) {
processChildren(child);
}
}
} catch (Throwable t) {
ExceptionUtils.handleThrowable(t);
log.error(sm.getString("containerBase.backgroundProcess.error"), t);
} finally {
if (container instanceof Context) {
((Context) container).unbind(false, originalClassLoader);
}
}
}
}

这个触发方式是在 ContainerBase 里的

1
2
3
4
5
6
7
8
protected class ContainerBackgroundProcessorMonitor implements Runnable {
@Override
public void run() {
if (getState().isAvailable()) {
threadStart();
}
}
}

而在这个 threadStart 里

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
protected void threadStart() {
if (backgroundProcessorDelay > 0
&& (getState().isAvailable() || LifecycleState.STARTING_PREP.equals(getState()))
&& (backgroundProcessorFuture == null || backgroundProcessorFuture.isDone())) {
if (backgroundProcessorFuture != null && backgroundProcessorFuture.isDone()) {
// There was an error executing the scheduled task, get it and log it
try {
backgroundProcessorFuture.get();
} catch (InterruptedException | ExecutionException e) {
log.error(sm.getString("containerBase.backgroundProcess.error"), e);
}
}
backgroundProcessorFuture = Container.getService(this).getServer().getUtilityExecutor()
.scheduleWithFixedDelay(new ContainerBackgroundProcessor(),
backgroundProcessorDelay, backgroundProcessorDelay,
TimeUnit.SECONDS);
}
}

就调用了线程池的 scheduleWithFixedDelay 方法提交了这个 ContainerBackgroundProcessor
仔细看代码会发现,

1
2
3
4
5
6
7
8
9
10
11
12
13
14
public StandardEngine() {

super();
pipeline.setBasic(new StandardEngineValve());
/* Set the jmvRoute using the system property jvmRoute */
try {
setJvmRoute(System.getProperty("jvmRoute"));
} catch(Exception ex) {
log.warn(sm.getString("standardEngine.jvmRouteFail"));
}
// By default, the engine will hold the reloading thread
backgroundProcessorDelay = 10;

}

这个就不用开启后台热加载,而主要的热加载同学应该是
org.apache.catalina.core.StandardContext#backgroundProcess

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
public void backgroundProcess() {

if (!getState().isAvailable()) {
return;
}

Loader loader = getLoader();
if (loader != null) {
try {
// 这里就用了 loader 的 backgroundProcess
loader.backgroundProcess();
} catch (Exception e) {
log.warn(sm.getString(
"standardContext.backgroundProcess.loader", loader), e);
}
}
Manager manager = getManager();
if (manager != null) {
try {
manager.backgroundProcess();
} catch (Exception e) {
log.warn(sm.getString(
"standardContext.backgroundProcess.manager", manager),
e);
}
}
WebResourceRoot resources = getResources();
if (resources != null) {
try {
resources.backgroundProcess();
} catch (Exception e) {
log.warn(sm.getString(
"standardContext.backgroundProcess.resources",
resources), e);
}
}
InstanceManager instanceManager = getInstanceManager();
if (instanceManager != null) {
try {
instanceManager.backgroundProcess();
} catch (Exception e) {
log.warn(sm.getString(
"standardContext.backgroundProcess.instanceManager",
resources), e);
}
}
super.backgroundProcess();
}

loader 的后台处理就是

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
@Override
public void backgroundProcess() {
if (reloadable && modified()) {
try {
Thread.currentThread().setContextClassLoader
(WebappLoader.class.getClassLoader());
if (context != null) {
context.reload();
}
} finally {
if (context != null && context.getLoader() != null) {
Thread.currentThread().setContextClassLoader
(context.getLoader().getClassLoader());
}
}
}
}

然后又会回到 context 的 reload,也就是 StandardContext 的 reload

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
@Override
public synchronized void reload() {

// Validate our current component state
if (!getState().isAvailable()) {
throw new IllegalStateException
(sm.getString("standardContext.notStarted", getName()));
}

if(log.isInfoEnabled()) {
log.info(sm.getString("standardContext.reloadingStarted",
getName()));
}

// Stop accepting requests temporarily.
setPaused(true);

try {
stop();
} catch (LifecycleException e) {
log.error(
sm.getString("standardContext.stoppingContext", getName()), e);
}

try {
start();
} catch (LifecycleException e) {
log.error(
sm.getString("standardContext.startingContext", getName()), e);
}

setPaused(false);

if(log.isInfoEnabled()) {
log.info(sm.getString("standardContext.reloadingCompleted",
getName()));
}

}

这样就是线程池结合后台处理,还是有些复杂的。

0%