Skip to content

Commit 644dd3a

Browse files
janhoyjanhoy
committed
SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
(cherry picked from commit 1553475)
1 parent 36759d5 commit 644dd3a

23 files changed

+522
-56
lines changed

‎solr/CHANGES.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ Other Changes
104104
* SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting
105105
cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
106106

107+
* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
108+
107109
================== 9.4.0 ==================
108110
New Features
109111
---------------------

‎solr/core/build.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,8 @@ dependencies {
161161

162162
compileOnly 'com.github.stephenc.jcip:jcip-annotations'
163163

164+
implementation 'com.j256.simplemagic:simplemagic'
165+
164166
// -- Test Dependencies
165167

166168
testRuntimeOnly 'org.slf4j:jcl-over-slf4j'

‎solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.solr.common.cloud.SolrZkClient;
2828
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
2929
import org.apache.solr.core.ConfigSetService;
30+
import org.apache.solr.util.FileTypeMagicUtil;
3031
import org.slf4j.Logger;
3132
import org.slf4j.LoggerFactory;
3233

@@ -100,6 +101,7 @@ public void runImpl(CommandLine cli) throws Exception {
100101
+ cli.getOptionValue("confname")
101102
+ " to ZooKeeper at "
102103
+ zkHost);
104+
FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
103105
ZkMaintenanceUtils.uploadToZK(
104106
zkClient,
105107
confPath,

‎solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.ArrayList;
2323
import java.util.Collections;
2424
import java.util.List;
25+
import java.util.Locale;
2526
import java.util.Map;
2627
import java.util.Objects;
2728
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -39,6 +40,7 @@
3940
import org.apache.solr.core.CoreDescriptor;
4041
import org.apache.solr.core.SolrConfig;
4142
import org.apache.solr.core.SolrResourceLoader;
43+
import org.apache.solr.util.FileTypeMagicUtil;
4244
import org.apache.zookeeper.CreateMode;
4345
import org.apache.zookeeper.KeeperException;
4446
import org.apache.zookeeper.data.Stat;
@@ -199,6 +201,15 @@ public void uploadFileToConfig(
199201
try {
200202
if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
201203
log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
204+
} else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
205+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
206+
throw new SolrException(
207+
SolrException.ErrorCode.BAD_REQUEST,
208+
String.format(
209+
Locale.ROOT,
210+
"Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s",
211+
fileName,
212+
mimeType));
202213
} else {
203214
// if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
204215
zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
@@ -340,7 +351,15 @@ private void copyData(String fromZkFilePath, String toZkFilePath)
340351
} else {
341352
log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
342353
byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
343-
zkClient.makePath(toZkFilePath, data, true);
354+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
355+
zkClient.makePath(toZkFilePath, data, true);
356+
} else {
357+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
358+
log.warn(
359+
"Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}",
360+
fromZkFilePath,
361+
mimeType);
362+
}
344363
}
345364
}
346365

‎solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.solr.common.SolrException;
3838
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
3939
import org.apache.solr.common.util.Utils;
40+
import org.apache.solr.util.FileTypeMagicUtil;
4041
import org.slf4j.Logger;
4142
import org.slf4j.LoggerFactory;
4243

@@ -150,9 +151,17 @@ public void uploadFileToConfig(
150151
if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
151152
log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
152153
} else {
153-
Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
154-
if (!Files.exists(filePath) || overwriteOnExists) {
155-
Files.write(filePath, data);
154+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
155+
Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
156+
if (!Files.exists(filePath) || overwriteOnExists) {
157+
Files.write(filePath, data);
158+
}
159+
} else {
160+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
161+
log.warn(
162+
"Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}",
163+
fileName,
164+
mimeType);
156165
}
157166
}
158167
}
@@ -205,8 +214,17 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
205214
"Not including uploading file to config, as it is a forbidden type: {}",
206215
file.getFileName());
207216
} else {
208-
Files.copy(
209-
file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
217+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {
218+
Files.copy(
219+
file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
220+
} else {
221+
String mimeType =
222+
FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
223+
log.warn(
224+
"Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",
225+
file.getFileName(),
226+
mimeType);
227+
}
210228
}
211229
return FileVisitResult.CONTINUE;
212230
}

‎solr/core/src/java/org/apache/solr/core/backup/BackupManager.java

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.apache.solr.common.util.Utils;
4141
import org.apache.solr.core.ConfigSetService;
4242
import org.apache.solr.core.backup.repository.BackupRepository;
43+
import org.apache.solr.util.FileTypeMagicUtil;
4344
import org.apache.zookeeper.CreateMode;
4445
import org.apache.zookeeper.KeeperException;
4546
import org.slf4j.Logger;
@@ -349,8 +350,16 @@ private void downloadConfigToRepo(ConfigSetService configSetService, String conf
349350
if (data == null) {
350351
data = new byte[0];
351352
}
352-
try (OutputStream os = repository.createOutput(uri)) {
353-
os.write(data);
353+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
354+
try (OutputStream os = repository.createOutput(uri)) {
355+
os.write(data);
356+
}
357+
} else {
358+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
359+
log.warn(
360+
"Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",
361+
filePath,
362+
mimeType);
354363
}
355364
}
356365
} else {
@@ -379,7 +388,15 @@ private void uploadConfigToSolrCloud(
379388
// probably ok since the config file should be small.
380389
byte[] arr = new byte[(int) is.length()];
381390
is.readBytes(arr, 0, (int) is.length());
382-
configSetService.uploadFileToConfig(configName, filePath, arr, false);
391+
if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
392+
configSetService.uploadFileToConfig(configName, filePath, arr, false);
393+
} else {
394+
String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
395+
log.warn(
396+
"Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}",
397+
filePath,
398+
mimeType);
399+
}
383400
}
384401
}
385402
break;

‎solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.solr.core.CoreContainer;
2828
import org.apache.solr.request.SolrQueryRequest;
2929
import org.apache.solr.response.SolrQueryResponse;
30+
import org.apache.solr.util.FileTypeMagicUtil;
3031

3132
/**
3233
* V2 API for adding or updating a single file within a configset.
@@ -67,11 +68,13 @@ public void updateConfigSetFile(SolrQueryRequest req, SolrQueryResponse rsp) thr
6768
if (fixedSingleFilePath.charAt(0) == '/') {
6869
fixedSingleFilePath = fixedSingleFilePath.substring(1);
6970
}
71+
byte[] data = inputStream.readAllBytes();
7072
if (fixedSingleFilePath.isEmpty()) {
7173
throw new SolrException(
7274
SolrException.ErrorCode.BAD_REQUEST,
7375
"The file path provided for upload, '" + singleFilePath + "', is not valid.");
74-
} else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
76+
} else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)
77+
|| FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
7578
throw new SolrException(
7679
SolrException.ErrorCode.BAD_REQUEST,
7780
"The file type provided for upload, '"
@@ -87,8 +90,7 @@ public void updateConfigSetFile(SolrQueryRequest req, SolrQueryResponse rsp) thr
8790
// For creating the baseNode, the cleanup parameter is only allowed to be true when
8891
// singleFilePath is not passed.
8992
createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);
90-
configSetService.uploadFileToConfig(
91-
configSetName, fixedSingleFilePath, inputStream.readAllBytes(), allowOverwrite);
93+
configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, allowOverwrite);
9294
}
9395
}
9496
}

‎solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.solr.util;
19+
20+
import com.j256.simplemagic.ContentInfo;
21+
import com.j256.simplemagic.ContentInfoUtil;
22+
import com.j256.simplemagic.ContentType;
23+
import java.io.ByteArrayInputStream;
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.nio.file.FileVisitResult;
27+
import java.nio.file.Files;
28+
import java.nio.file.Path;
29+
import java.nio.file.SimpleFileVisitor;
30+
import java.nio.file.attribute.BasicFileAttributes;
31+
import java.util.Arrays;
32+
import java.util.HashSet;
33+
import java.util.Locale;
34+
import java.util.Set;
35+
import org.apache.solr.common.SolrException;
36+
37+
/** Utility class to guess the mime type of file based on its magic number. */
38+
public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
39+
private final ContentInfoUtil util;
40+
private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", ".."));
41+
42+
public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
43+
44+
FileTypeMagicUtil() {
45+
try {
46+
util = new ContentInfoUtil("/magic/executables", this);
47+
} catch (IOException e) {
48+
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e);
49+
}
50+
}
51+
52+
/**
53+
* Asserts that an entire configset folder is legal to upload.
54+
*
55+
* @param confPath the path to the folder
56+
* @throws SolrException if an illegal file is found in the folder structure
57+
*/
58+
public static void assertConfigSetFolderLegal(Path confPath) throws IOException {
59+
Files.walkFileTree(
60+
confPath,
61+
new SimpleFileVisitor<Path>() {
62+
@Override
63+
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
64+
throws IOException {
65+
// Read first 100 bytes of the file to determine the mime type
66+
try (InputStream fileStream = Files.newInputStream(file)) {
67+
byte[] bytes = new byte[100];
68+
fileStream.read(bytes);
69+
if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
70+
throw new SolrException(
71+
SolrException.ErrorCode.BAD_REQUEST,
72+
String.format(
73+
Locale.ROOT,
74+
"Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
75+
file,
76+
FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
77+
}
78+
return FileVisitResult.CONTINUE;
79+
}
80+
}
81+
82+
@Override
83+
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
84+
throws IOException {
85+
if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
86+
return FileVisitResult.SKIP_SUBTREE;
87+
88+
return FileVisitResult.CONTINUE;
89+
}
90+
});
91+
}
92+
93+
/**
94+
* Guess the mime type of file based on its magic number.
95+
*
96+
* @param stream input stream of the file
97+
* @return string with content-type or "application/octet-stream" if unknown
98+
*/
99+
public String guessMimeType(InputStream stream) {
100+
try {
101+
ContentInfo info = util.findMatch(stream);
102+
if (info == null) {
103+
return ContentType.OTHER.getMimeType();
104+
}
105+
return info.getContentType().getMimeType();
106+
} catch (IOException e) {
107+
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
108+
}
109+
}
110+
111+
/**
112+
* Guess the mime type of file bytes based on its magic number.
113+
*
114+
* @param bytes the first bytes at start of the file
115+
* @return string with content-type or "application/octet-stream" if unknown
116+
*/
117+
public String guessMimeType(byte[] bytes) {
118+
return guessMimeType(new ByteArrayInputStream(bytes));
119+
}
120+
121+
@Override
122+
public void error(String line, String details, Exception e) {
123+
throw new SolrException(
124+
SolrException.ErrorCode.SERVER_ERROR,
125+
String.format(Locale.ROOT, "%s: %s", line, details),
126+
e);
127+
}
128+
129+
/**
130+
* Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
131+
* are:
132+
*
133+
* <ul>
134+
* <li><code>application/x-java-applet</code>: java class file
135+
* <li><code>application/zip</code>: jar or zip archives
136+
* <li><code>application/x-tar</code>: tar archives
137+
* <li><code>text/x-shellscript</code>: shell or bash script
138+
* </ul>
139+
*
140+
* @param fileStream stream from the file content
141+
* @return true if file is among the forbidden mime-types
142+
*/
143+
public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
144+
return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
145+
}
146+
147+
/**
148+
* Determine forbidden file type based on magic bytes matching of the first bytes of the file.
149+
*
150+
* @param bytes byte array of the file content
151+
* @return true if file is among the forbidden mime-types
152+
*/
153+
public static boolean isFileForbiddenInConfigset(byte[] bytes) {
154+
if (bytes == null || bytes.length == 0)
155+
return false; // A ZK znode may be a folder with no content
156+
return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
157+
}
158+
159+
private static final Set<String> forbiddenTypes =
160+
new HashSet<>(
161+
Arrays.asList(
162+
System.getProperty(
163+
"solr.configset.upload.mimetypes.forbidden",
164+
"application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
165+
.split(",")));
166+
}

0 commit comments

Comments
 (0)