Skip to content

Commit 7e9a2e6

Browse files
janhoyjanhoy
committed
SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
1 parent 6e9ed20 commit 7e9a2e6

20 files changed

+410
-11
lines changed

‎lucene/ivy-versions.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ com.fasterxml.jackson.core.version = 2.15.2
7171
/com.healthmarketscience.jackcess/jackcess = 3.0.1
7272
/com.healthmarketscience.jackcess/jackcess-encrypt = 3.0.0
7373
/com.ibm.icu/icu4j = 62.1
74+
/com.j256.simplemagic/simplemagic = 1.17
7475
/com.jayway.jsonpath/json-path = 2.7.0
7576
/com.lmax/disruptor = 3.4.2
7677
/com.pff/java-libpst = 0.9.3

‎solr/CHANGES.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ Other Changes
6464
* SOLR-14853: Security: Converted enableRemoteStreaming and enableStreamBody solrconfig options into system properties and env vars.
6565
Attempts to set them the old way are no-op and log a warning. (David Smiley, janhoy, Ishan Chattopadhyaya)
6666

67+
* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
68+
6769
================== 8.11.2 ==================
6870

6971
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

‎solr/core/ivy.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
<dependency org="com.google.guava" name="failureaccess" rev="${/com.google.guava/failureaccess}" conf="compile"/>
4646
<dependency org="com.google.guava" name="listenablefuture" rev="${/com.google.guava/listenablefuture}" conf="compile"/>
4747
<dependency org="com.google.j2objc" name="j2objc-annotations" rev="${/com.google.j2objc/j2objc-annotations}" conf="compile"/>
48+
<dependency org="com.j256.simplemagic" name="simplemagic" rev="${/com.j256.simplemagic/simplemagic}" conf="compile"/>
4849
<dependency org="org.locationtech.spatial4j" name="spatial4j" rev="${/org.locationtech.spatial4j/spatial4j}" conf="compile"/>
4950
<dependency org="org.antlr" name="antlr4-runtime" rev="${/org.antlr/antlr4-runtime}"/>
5051
<dependency org="org.apache.commons" name="commons-math3" rev="${/org.apache.commons/commons-math3}" conf="compile"/>

‎solr/core/src/java/org/apache/solr/core/backup/BackupManager.java

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.List;
2828
import java.util.Objects;
2929
import java.util.Optional;
30+
import org.apache.solr.util.FileTypeMagicUtil;
3031

3132
import com.google.common.base.Preconditions;
3233
import org.apache.lucene.store.IOContext;
@@ -302,8 +303,16 @@ private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throw
302303
if (children.size() == 0) {
303304
log.debug("Writing file {}", file);
304305
byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
305-
try (OutputStream os = repository.createOutput(repository.resolve(dir, file))) {
306-
os.write(data);
306+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
307+
try (OutputStream os = repository.createOutput(repository.resolve(dir, file))) {
308+
os.write(data);
309+
}
310+
} else {
311+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
312+
log.warn(
313+
"Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",
314+
file,
315+
mimeType);
307316
}
308317
} else {
309318
URI uri = repository.resolve(dir, file);
@@ -329,7 +338,15 @@ private void uploadToZk(SolrZkClient zkClient, URI sourceDir, String destZkPath)
329338
try (IndexInput is = repository.openInput(sourceDir, file, IOContext.DEFAULT)) {
330339
byte[] arr = new byte[(int) is.length()]; // probably ok since the config file should be small.
331340
is.readBytes(arr, 0, (int) is.length());
332-
zkClient.makePath(zkNodePath, arr, true);
341+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
342+
zkClient.makePath(zkNodePath, arr, true);
343+
} else {
344+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
345+
log.warn(
346+
"Not restoring configset file {} to zookeeper, as it matched the MAGIC signature of a forbidden mime type {}",
347+
file,
348+
mimeType);
349+
}
333350
} catch (KeeperException | InterruptedException e) {
334351
throw new IOException(SolrZkClient.checkInterrupted(e));
335352
}

‎solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.HashSet;
2525
import java.util.Iterator;
2626
import java.util.List;
27+
import java.util.Locale;
2728
import java.util.Map;
2829
import java.util.Set;
2930
import java.util.concurrent.TimeUnit;
@@ -56,6 +57,7 @@
5657
import org.apache.solr.security.AuthenticationPlugin;
5758
import org.apache.solr.security.AuthorizationContext;
5859
import org.apache.solr.security.PermissionNameProvider;
60+
import org.apache.solr.util.FileTypeMagicUtil;
5961
import org.apache.zookeeper.CreateMode;
6062
import org.apache.zookeeper.KeeperException;
6163
import org.slf4j.Logger;
@@ -203,9 +205,17 @@ private void handleConfigUploadRequest(SolrQueryRequest req, SolrQueryResponse r
203205
try {
204206
// Create a node for the configuration in zookeeper
205207
// For creating the baseZnode, the cleanup parameter is only allowed to be true when singleFilePath is not passed.
206-
createBaseZnode(zkClient, overwritesExisting, requestIsTrusted, configPathInZk);
207-
String filePathInZk = configPathInZk + "/" + fixedSingleFilePath;
208-
zkClient.makePath(filePathInZk, IOUtils.toByteArray(inputStream), CreateMode.PERSISTENT, null, !allowOverwrite, true);
208+
byte[] bytes = IOUtils.toByteArray(inputStream);
209+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
210+
createBaseZnode(zkClient, overwritesExisting, requestIsTrusted, configPathInZk);
211+
String filePathInZk = configPathInZk + "/" + fixedSingleFilePath;
212+
zkClient.makePath(filePathInZk, bytes, CreateMode.PERSISTENT, null, !allowOverwrite, true);
213+
} else {
214+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(bytes);
215+
throw new SolrException(ErrorCode.BAD_REQUEST,
216+
String.format(Locale.ROOT, "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
217+
fixedSingleFilePath, mimeType));
218+
}
209219
} catch(KeeperException.NodeExistsException nodeExistsException) {
210220
throw new SolrException(ErrorCode.BAD_REQUEST,
211221
"The path " + singleFilePath + " for configSet " + configSetName + " already exists. In order to overwrite, provide overwrite=true or use an HTTP PUT with the V2 API.");
@@ -244,8 +254,15 @@ private void handleConfigUploadRequest(SolrQueryRequest req, SolrQueryResponse r
244254
if (zipEntry.isDirectory()) {
245255
zkClient.makePath(filePathInZk, false, true);
246256
} else {
247-
createZkNodeIfNotExistsAndSetData(zkClient, filePathInZk,
248-
IOUtils.toByteArray(zis));
257+
byte[] bytes = IOUtils.toByteArray(zis);
258+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
259+
createZkNodeIfNotExistsAndSetData(zkClient, filePathInZk, bytes);
260+
} else {
261+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(bytes);
262+
throw new SolrException(ErrorCode.BAD_REQUEST,
263+
String.format(Locale.ROOT, "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
264+
zipEntry.getName(), mimeType));
265+
}
249266
}
250267
}
251268
zis.close();

‎solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.solr.util;
19+
20+
import com.j256.simplemagic.ContentInfo;
21+
import com.j256.simplemagic.ContentInfoUtil;
22+
import com.j256.simplemagic.ContentType;
23+
import java.io.ByteArrayInputStream;
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.nio.file.FileVisitResult;
27+
import java.nio.file.Files;
28+
import java.nio.file.Path;
29+
import java.nio.file.SimpleFileVisitor;
30+
import java.nio.file.attribute.BasicFileAttributes;
31+
import java.util.Arrays;
32+
import java.util.HashSet;
33+
import java.util.Locale;
34+
import java.util.Set;
35+
import org.apache.solr.common.SolrException;
36+
37+
/** Utility class to guess the mime type of file based on its magic number. */
38+
public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
39+
private final ContentInfoUtil util;
40+
private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", ".."));
41+
42+
public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
43+
44+
FileTypeMagicUtil() {
45+
try {
46+
util = new ContentInfoUtil("/magic/executables", this);
47+
} catch (IOException e) {
48+
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e);
49+
}
50+
}
51+
52+
/**
53+
* Asserts that an entire configset folder is legal to upload.
54+
*
55+
* @param confPath the path to the folder
56+
* @throws SolrException if an illegal file is found in the folder structure
57+
*/
58+
public static void assertConfigSetFolderLegal(Path confPath) throws IOException {
59+
Files.walkFileTree(confPath, new SimpleFileVisitor<Path>() {
60+
@Override
61+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
62+
// Read first 100 bytes of the file to determine the mime type
63+
try(InputStream fileStream = Files.newInputStream(file)) {
64+
byte[] bytes = new byte[100];
65+
fileStream.read(bytes);
66+
if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
67+
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
68+
String.format(Locale.ROOT, "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
69+
file, FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
70+
}
71+
return FileVisitResult.CONTINUE;
72+
}
73+
}
74+
75+
@Override
76+
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
77+
if (SKIP_FOLDERS.contains(dir.getFileName().toString())) return FileVisitResult.SKIP_SUBTREE;
78+
79+
return FileVisitResult.CONTINUE;
80+
}
81+
});
82+
}
83+
84+
/**
85+
* Guess the mime type of file based on its magic number.
86+
*
87+
* @param stream input stream of the file
88+
* @return string with content-type or "application/octet-stream" if unknown
89+
*/
90+
public String guessMimeType(InputStream stream) {
91+
try {
92+
ContentInfo info = util.findMatch(stream);
93+
if (info == null) {
94+
return ContentType.OTHER.getMimeType();
95+
}
96+
return info.getContentType().getMimeType();
97+
} catch (IOException e) {
98+
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
99+
}
100+
}
101+
102+
/**
103+
* Guess the mime type of file bytes based on its magic number.
104+
*
105+
* @param bytes the first bytes at start of the file
106+
* @return string with content-type or "application/octet-stream" if unknown
107+
*/
108+
public String guessMimeType(byte[] bytes) {
109+
return guessMimeType(new ByteArrayInputStream(bytes));
110+
}
111+
112+
@Override
113+
public void error(String line, String details, Exception e) {
114+
throw new SolrException(
115+
SolrException.ErrorCode.SERVER_ERROR,
116+
String.format(Locale.ROOT, "%s: %s", line, details),
117+
e);
118+
}
119+
120+
/**
121+
* Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
122+
* are:
123+
*
124+
* <ul>
125+
* <li><code>application/x-java-applet</code>: java class file
126+
* <li><code>application/zip</code>: jar or zip archives
127+
* <li><code>application/x-tar</code>: tar archives
128+
* <li><code>text/x-shellscript</code>: shell or bash script
129+
* </ul>
130+
*
131+
* @param fileStream stream from the file content
132+
* @return true if file is among the forbidden mime-types
133+
*/
134+
public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
135+
return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
136+
}
137+
138+
/**
139+
* Determine forbidden file type based on magic bytes matching of the first bytes of the file.
140+
*
141+
* @param bytes byte array of the file content
142+
* @return true if file is among the forbidden mime-types
143+
*/
144+
public static boolean isFileForbiddenInConfigset(byte[] bytes) {
145+
return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
146+
}
147+
148+
private static final Set<String> forbiddenTypes =
149+
new HashSet<>(
150+
Arrays.asList(
151+
System.getProperty(
152+
"solr.configset.upload.mimetypes.forbidden",
153+
"application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
154+
.split(",")));
155+
156+
}

‎solr/core/src/java/org/apache/solr/util/SolrCLI.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,6 +1971,7 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th
19711971

19721972
echoIfVerbose("Uploading " + confPath.toAbsolutePath().toString() +
19731973
" for config " + confname + " to ZooKeeper at " + cloudSolrClient.getZkHost(), cli);
1974+
FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
19741975
((ZkClientClusterStateProvider) cloudSolrClient.getClusterStateProvider()).uploadConfig(confPath, confname);
19751976
}
19761977

@@ -2265,6 +2266,7 @@ protected void runImpl(CommandLine cli) throws Exception {
22652266
echo("Uploading " + confPath.toAbsolutePath().toString() +
22662267
" for config " + cli.getOptionValue("confname") + " to ZooKeeper at " + zkHost);
22672268

2269+
FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
22682270
zkClient.upConfig(confPath, confName);
22692271
} catch (Exception e) {
22702272
log.error("Could not complete upconfig operation for reason: ", e);

‎solr/core/src/resources/magic/executables

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# POSIX tar archives
2+
# URL: https://en.wikipedia.org/wiki/Tar_(computing)
3+
# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current
4+
# header mainly padded with nul bytes
5+
500 quad 0
6+
!:strength /2
7+
# filename or extended attribute printable strings in range space null til umlaut ue
8+
>0 ubeshort >0x1F00
9+
>>0 ubeshort <0xFCFD
10+
# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad
11+
# at https://sourceforge.net/projects/s-tar/files/testscripts/
12+
>>>508 ubelong&0x8B9E8DFF 0
13+
# nul, space or ascii digit 0-7 at start of mode
14+
>>>>100 ubyte&0xC8 =0
15+
>>>>>101 ubyte&0xC8 =0
16+
# nul, space at end of check sum
17+
>>>>>>155 ubyte&0xDF =0
18+
# space or ascii digit 0 at start of check sum
19+
>>>>>>>148 ubyte&0xEF =0x20
20+
# check for specific 1st member name that indicates other mime type and file name suffix
21+
>>>>>>>>0 string TpmEmuTpms/permall
22+
!:mime application/x-tar
23+
!:ext tar
24+
# other stuff in padding
25+
# some implementations add new fields to the blank area at the end of the header record
26+
# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option
27+
>>257 ulong !0 tar archive (old)
28+
!:mime application/x-tar
29+
!:ext tar
30+
# magic in newer, GNU, posix variants
31+
>257 string =ustar
32+
# 2 last char of magic and UStar version because string expression does not work
33+
# 2 space characters followed by a null for GNU variant
34+
>>261 ubelong =0x72202000 POSIX tar archive (GNU)
35+
!:mime application/x-gtar
36+
!:ext tar/gtar
37+
38+
39+
# Zip archives (Greg Roelofs, c/o [email protected])
40+
0 string PK\005\006 Zip archive data (empty)
41+
0 string PK\003\004 Zip archive data
42+
!:strength +1
43+
!:mime application/zip
44+
!:ext zip/cbz
45+
46+
47+
# JAVA
48+
0 belong 0xcafebabe
49+
>4 ubelong >30 compiled Java class data,
50+
!:mime application/x-java-applet
51+
#!:mime application/java-byte-code
52+
!:ext class
53+
54+
55+
# SHELL scripts
56+
#0 string/w : shell archive or script for antique kernel text
57+
0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable
58+
!:mime text/x-shellscript
59+
!:ext sh/bash

‎solr/core/src/test-files/magic/HelloWorld.java.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
class HelloWorld {
2+
public static void main(String[] args) {
3+
System.out.println("Hellow world");
4+
}
5+
}

‎solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin

426 Bytes
Binary file not shown.

‎solr/core/src/test-files/magic/README.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
The two binary files were created by the following commands:
2+
3+
```bash
4+
echo "Hello" > hello.txt && \
5+
tar -cvf hello.tar.bin hello.txt && \
6+
rm hello.txt
7+
8+
cp HelloWorld.java.txt HelloWorld.java && \
9+
javac HelloWorld.java && \
10+
mv HelloWorld.class HelloWorldJavaClass.class.bin && \
11+
rm HelloWorld.java
12+
```

‎solr/core/src/test-files/magic/hello.tar.bin

4 KB
Binary file not shown.

‎solr/core/src/test-files/magic/plain.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello world

‎solr/core/src/test-files/magic/shell.sh.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#! /usr/bin/env bash
2+
echo Hello

0 commit comments

Comments
 (0)