I've got some issue while reading ZipInputStream to byte array - result is -1. However I can read it as file stream, so I'm not sure if I'm missing something:
public void readContent () throws IOException {
int size = 2048;
byte[] file = new byte[size];
byte[] stream = new byte[size];
FileInputStream fileStream = new FileInputStream(this.file);
ZipInputStream zipStream = new ZipInputStream(fileStream);;
System.out.println("Reading as a file stream " + fileStream.read(file,0,size)); //output 2048
System.out.println("Reading as a zip stream " + zipStream.read(stream,0,size)); // output -1
}
Data from ZipInputStream should be read via ZipEntry. This is example how to read zip archive from given InputStream and retrieve Map with key - full zip entry path, value - zip entry data:
public class ZipUtils {
private static final Comparator<String> COMPARATOR_STRING_DESC = (str1, str2) -> str1 == null ? 1 : str2 == null ? -1 : -str1.compareTo(str2);
public static Map<String, byte[]> unzipIt(InputStream is) throws IOException {
try (ZipInputStream in = new ZipInputStream(is)) {
ZipEntry entry;
Map<String, byte[]> content = new HashMap<>();
Set<String> dirs = new TreeSet<>(COMPARATOR_STRING_DESC);
while ((entry = in.getNextEntry()) != null) {
String path = removeDirectoryMarker(replaceIncorrectFileSeparators(entry.getName()));
if (isDirectory(entry)) {
dirs.add(path);
} else {
content.put(path, IOUtils.toByteArray(in));
}
}
addOnlyEmptyDirectories(dirs, content);
return content.isEmpty() ? Collections.emptyMap() : content;
}
}
private static boolean isDirectory(ZipEntry entry) {
return entry.isDirectory() || entry.getName().endsWith(ILLEGAL_DIR_MARKER);
}
private static void addOnlyEmptyDirectories(Set<String> dirs, Map<String, byte[]> content) {
if (dirs.isEmpty()) {
return;
}
Set<String> paths = new HashSet<>(content.keySet());
for (String dir : dirs) {
boolean empty = true;
for (String path : paths) {
if (path.startsWith(dir)) {
empty = false;
break;
}
}
if (empty) {
content.put(dir, null);
}
}
}
private static final String DIR_MARKER = "/";
private static final String ILLEGAL_DIR_MARKER = "\\";
private static final java.util.regex.Pattern BACK_SLASH = Pattern.compile("\\\\");
private static String removeDirectoryMarker(String path) {
return path.endsWith(DIR_MARKER) || path.endsWith(ILLEGAL_DIR_MARKER) ? path.substring(0, path.length() - 1) : path;
}
private static String replaceIncorrectFileSeparators(String path) {
return BACK_SLASH.matcher(path).replaceAll(DIR_MARKER);
}
}
Related
I don't know why the isFile() of the element f in the files list traversed by the for loop in the FileInit class in the code below is always false.false. But if delete the content related to FileItem, it will return to normal.It's kind of weird to me, and I didn't find a targeted answer. It might be some peculiarity of Java.A cloud disk project is used to learn Java, using Google's Gson library.
public class FileInit {
String path;
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
//Query file method, will return a list containing file information
public String queryFiles() throws NoSuchAlgorithmException, IOException {
//Query the path and build a File collection
File file = new File(path);
File[] files = file.listFiles();
//Build a List to store the converted FileItem
List<FileItem> fileItems = new ArrayList<>();
//Traversing files to make judgments
for (File f:files){
FileItem fileItem = new FileItem(f);
fileItem.printFile();
fileItems.add(fileItem);
}
Gson gson = new Gson();
return gson.toJson(fileItems);
}
//formatted output
public void printFiles(String files){
Gson gson = new Gson();
Type type = new TypeToken<List<FileItem>>(){}.getType();
List<FileItem> fileItems = gson.fromJson(files, type);
// Format output file list
int count = 0;
for (FileItem f :fileItems ) {
String name;
if (f.getFileType()==null){
name = f.getFileName() + "/";
}else {
name = f.getFileName();
}
System.out.printf("%-40s", name);
count++;
if (count % 3 == 0) {
System.out.println();
}
}
System.out.println();
}
//Change directory command
public void changeDic(String addPath){
File fileDic1 = new File(path+addPath);
File fileDic2 = new File(addPath);
if (addPath.equals("..")) {
File parent = new File(path).getParentFile();
if (parent != null) {
this.path = parent.getPath();
}else{
System.out.println("Parent directory does not exist");
}
}
else {
if (fileDic1.exists()){
this.path = path+addPath;
} else if (fileDic2.exists()) {
this.path = addPath;
}else{
System.out.println("Illegal input path");
}
}
}
}
public class FileItem {
private String fileName;
private String fileHash;
private String filePath;
private long fileLength;
private String fileType;
//Construction method of FileItem
/*
Build a construction method that only needs fileName and filePath, and judge whether to calculate the hash value, file size, and file type according to the folder or file
*/
public FileItem(String fileName,String filePath) {
this.fileName = fileName;
this.filePath = filePath;
File file =new File(this.filePath+"/"+this.fileName);
if (file.isFile()){
try {
//Get the file size through the file built-in method
this.fileLength = file.length();
// Define a regular expression for extracting the suffix of the file name
String regex = "\\.(\\w+)$";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(fileName);
// If the match is successful
if (matcher.find()) {
// Get the suffix of the file name
this.fileType = matcher.group(1);
}else{
this.fileType = null;
}
//Calculate the Hash value of the file by calling the FileHash method
this.fileHash=FileHash(file.getPath());
System.out.printf(fileHash);
System.out.print("\n");
} catch (NoSuchAlgorithmException | IOException e) {
throw new RuntimeException(e);
}
}else{
this.fileName=fileName;
this.fileLength=0;
this.fileType = null;
this.fileHash=null;
}
}
//Build a constructor that only needs a json file
public FileItem(String json){
Gson gson = new Gson();
FileItem fileItem = gson.fromJson(json,FileItem.class);
this.fileName=fileItem.getFileName();
this.filePath=fileItem.getFilePath();
this.fileHash=fileItem.getFileHash();
this.fileLength=fileItem.getFileLength();
this.fileType=fileItem.getFileType();
}
//Realize mutual conversion between FileItem and File class
public File toFile(){
return new File(this.filePath+"/"+this.fileName);
}
public FileItem(File file) throws NoSuchAlgorithmException, IOException {
FileItem fileItem = new FileItem(file.getName(),file.getPath());
this.fileName=fileItem.getFileName();
this.filePath=fileItem.getFilePath();
this.fileHash=fileItem.getFileHash();
this.fileLength=fileItem.getFileLength();
this.fileType=fileItem.getFileType();
}
//Display FileItem related information
public void printFile(){
if (fileType==null){
System.out.println(fileName+"是文件夹");
}else {
System.out.println(fileName+"是"+fileType+"文件");
}
double fileLenOp=(double)fileLength;
//Different file sizes use different output methods
if(fileLength<=1024*1024){
System.out.printf("file size is:%.3fKB\n",fileLenOp/1024);
} else if ((fileLength<1024*1024*1024)) {
System.out.printf("file size is:%.3fMB\n",fileLenOp/1024/1024);
}else {
System.out.printf("file size is:%.3fGB\n",fileLenOp/1024/1024/1024);
}
System.out.println("file hash is:"+fileHash);
}
//Getter and Setter methods for FileItem
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getFileHash() {
return fileHash;
}
public void setFileHash(String fileHash) {
this.fileHash = fileHash;
}
public String getFilePath() {
return filePath;
}
public void setFilePath(String filePath) {
this.filePath = filePath;
}
public long getFileLength() {
return fileLength;
}
public void setFileLength(long fileLength) {
this.fileLength = fileLength;
}
public String getFileType() {
return fileType;
}
public void setFileType(String fileType) {
this.fileType = fileType;
}
}
I try to query the file information in a folder by using the methods related to the File class. And return them as a collection. Then realize the conversion of the File class and the FileItem class by traversing this collection to form a new FileItem class collection , so that it can be converted into Json format data for network transmission. However, all fileitems in the generated FileItem collection are judged as folders. And if you comment out the FileItem related content, the judgment of File will be normal, but if you don’t do this , the judgment of File will always remain false.
I'm running into a problem using the commons compress library to create a tar.gz of a directory. I have a directory structure that is as follows.
parent/
child/
file1.raw
file2.raw
file3.raw
I hope the compressed structure is like this.
child/
file1.raw
file2.raw
file3.raw
Is there any way to remove the outermost layer during compression?
I've seen such examples, but I can't work properly,and can only handle fixed name structures
public static void main(String[] args) throws IOException {
String hallFilePath = "E:/" + "packs";
compress(Paths.get(hallFilePath).toString(), hallFilePath + ".zip");
}
public static void compress(String fromPath, String toPath) throws IOException {
File fromFile = new File(fromPath);
File toFile = new File(toPath);
if (!fromFile.exists()) {
throw new ServiceException(fromPath + "不存在！");
}
try (FileOutputStream outputStream = new FileOutputStream(toFile); CheckedOutputStream checkedOutputStream = new CheckedOutputStream(outputStream, new CRC32()); ZipOutputStream zipOutputStream = new ZipOutputStream(checkedOutputStream)) {
String baseDir = "";
compress(fromFile, zipOutputStream, baseDir);
}
}
private static void compress(File file, ZipOutputStream zipOut, String baseDir) throws IOException {
if (file.isDirectory()) {
compressDirectory(file, zipOut, baseDir);
} else {
if (baseDir.equals("packs" + File.separator)) {
baseDir = File.separator;
} else if (baseDir.equals("packs" + File.separator + "examineeInfo" + File.separator)) {
baseDir = "examineeInfo" + File.separator;
}
compressFile(file, zipOut, baseDir);
}
}
private static void compressFile(File file, ZipOutputStream zipOut, String baseDir) throws IOException {
if (!file.exists()) {
return;
}
try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
ZipEntry entry = new ZipEntry(baseDir + file.getName());
zipOut.putNextEntry(entry);
int count;
byte[] data = new byte[BUFFER];
while ((count = bis.read(data, 0, BUFFER)) != -1) {
zipOut.write(data, 0, count);
}
}
}
private static void compressDirectory(File dir, ZipOutputStream zipOut, String baseDir) throws IOException {
File[] files = dir.listFiles();
if (files != null && ArrayUtils.isNotEmpty(files)) {
for (File file : files) {
compress(file, zipOut, baseDir + dir.getName() + File.separator);
}
}
}
I've found the Resources.readLines() and Files.readLines() to be helpfull in simplifiying my code.
The problem is that I often read gzip-compressed txt-files or txt-files in zip archives from URL's (HTTP and FTP).
Is there a way to use Guava's methods to read from these URL's too? Or is that only possible with Java's GZIPInputStream/ZipInputStream?
You can create your own ByteSources:
For GZip:
public class GzippedByteSource extends ByteSource {
private final ByteSource source;
public GzippedByteSource(ByteSource gzippedSource) { source = gzippedSource; }
#Override public InputStream openStream() throws IOException {
return new GZIPInputStream(source.openStream());
}
}
Then use it:
Charset charset = ... ;
new GzippedByteSource(Resources.asByteSource(url)).toCharSource(charset).readLines();
Here is the implementation for the Zip. This assumes that you read only one entry.
public static class ZipEntryByteSource extends ByteSource {
private final ByteSource source;
private final String entryName;
public ZipEntryByteSource(ByteSource zipSource, String entryName) {
this.source = zipSource;
this.entryName = entryName;
}
#Override public InputStream openStream() throws IOException {
final ZipInputStream in = new ZipInputStream(source.openStream());
while (true) {
final ZipEntry entry = in.getNextEntry();
if (entry == null) {
in.close();
throw new IOException("No entry named " + entry);
} else if (entry.getName().equals(this.entryName)) {
return new InputStream() {
#Override
public int read() throws IOException {
return in.read();
}
#Override
public void close() throws IOException {
in.closeEntry();
in.close();
}
};
} else {
in.closeEntry();
}
}
}
}
And you can use it like this:
Charset charset = ... ;
String entryName = ... ; // Name of the entry inside the zip file.
new ZipEntryByteSource(Resources.asByteSource(url), entryName).toCharSource(charset).readLines();
As Olivier Grégoire said, you can create the necessary ByteSources for whatever compression scheme you need in order to use Guava's readLines function.
For zip archives though, although it's possible to do it, I don't think it's worth it. It will be easier to make your own readLines method that iterates over the zip entries and reads the lines of each entry on your own. Here's a class that demonstrates how to read and output the lines of a URL pointing at a zip archive:
public class ReadLinesOfZippedUrl {
public static List<String> readLines(String urlStr, Charset charset) {
List<String> retVal = new LinkedList<>();
try (ZipInputStream zipInputStream = new ZipInputStream(new URL(urlStr).openStream())) {
for (ZipEntry zipEntry = zipInputStream.getNextEntry(); zipEntry != null; zipEntry = zipInputStream.getNextEntry()) {
// don't close this reader or you'll close the underlying zip stream
BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream, charset));
retVal.addAll(reader.lines().collect(Collectors.toList())); // slurp all the lines from one entry
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return retVal;
}
public static void main(String[] args) {
String urlStr = "http://central.maven.org/maven2/com/google/guava/guava/18.0/guava-18.0-sources.jar";
Charset charset = StandardCharsets.UTF_8;
List<String> lines = readLines(urlStr, charset);
lines.forEach(System.out::println);
}
}
I'm downloading a zip file from an ftp server. The zip file contains a couple csv files. I'm trying to extract both csv files so that I can pass them into Opencsv, but I seem to be having some issues. I'm assuming there must be a better way to handle this than the way I'm doing it below. How do you return my csv files so that they are available in a list for my csv reader?
My code
ftp.retrieveFile(file, output);
InputStream inputStream = new ByteArrayInputStream(output.toByteArray());
Map<String, InputStream> inputStreams = new HashMap<>();
if (importTask.isZipfile()) {
inputStreams.put("products", importUtils.getZipData(new ZipInputStream(inputStream), importTask.getFilename()));
if(importTask.getCustomerFilename() != null) {
inputStream = new ByteArrayInputStream(output.toByteArray());
inputStreams.put("customers", importUtils.getZipData(new ZipInputStream(inputStream), importTask.getCustomerFilename()));
}
} else {
inputStreams.put("products", inputStream);
}
ftp.logout();
ftp.disconnect();
return inputStreams;
Zip
public InputStream getZipData(ZipInputStream zip, String filename) throws FileNotFoundException, IOException {
for (ZipEntry e; (e = zip.getNextEntry()) != null;) {
if (e.getName().equals(filename)) {
return zip;
}
}
throw new FileNotFoundException("zip://" + filename);
}
If you use Java 7+ you have an easier solution than that; you can just use the zip filesystem provider.
Here is some sample code; note that you need to .close() the generated InputStreams and FileSystems:
public static void getFsFromZipFile(final Path zipFile)
throws IOException
{
final URI uri = URI.create("jar:" + zipFile.toUri());
final Map<String, ?> env = Collections.singletonMap("readonly", "true");
return FileSystems.newFileSystem(uri, env);
}
public static getInputStreamFromZip(final FileSystem zipfs, final String name)
throws IOException
{
return Files.newInputStream(zipfs.getPath(name));
}
This is not how I'd recommend you do it however. What I'd recommend is this:
final Map<String, Path> getFilesFromZip(final Path zipFile, final String... names)
throws IOException
{
Path tmpfile;
final URI uri = URI.create("jar:" + zipFile.toUri());
final Map<String, ?> env = Collections.singletonMap("readonly", "true);
final Map<String, Path> ret = new HashMap<>();
try (
final FileSystem zipfs = FileSystems.newFileSystem(uri, env);
) {
for (final String name: names) {
tmpfile = Files.createTempFile("tmp", ".csv");
Files.copy(zipfs.getPath(name), tmpfile);
ret.put(name, tmpfile);
}
return ret;
}
}
I'd like to write a function that deletes all empty folders, with the option to ignore certain file types (allowed file types are stored in the hashmap) and tell if it should look inside directories.
Calling:
HashMap<String, Boolean> allowedFileTypes = new HashMap<String, Boolean>();
allowedFileTypes.put("pdf", true);
deleteEmptyFolders("ABSOLUTE PATH", allowedFileTypes, true);
Function:
public static void deleteEmptyFolders(String folderPath, HashMap<String, Boolean> allowedFileTypes, boolean followDirectory) {
File targetFolder = new File(folderPath);
File[] allFiles = targetFolder.listFiles();
if (allFiles.length == 0)
targetFolder.delete();
else {
boolean importantFiles = false;
for (File file : allFiles) {
String fileType = "folder";
if (!file.isDirectory())
fileType = file.getName().substring(file.getName().lastIndexOf('.') + 1);
if (!importantFiles)
importantFiles = (allowedFileTypes.get(fileType) != null);
if (file.isDirectory() && followDirectory)
deleteEmptyFolders(file.getAbsolutePath(), allowedFileTypes, followDirectory);
}
// if there are no important files in the target folder
if (!importantFiles)
targetFolder.delete();
}
}
The problem is that nothing is happening, even though it looks through all folders till the end. Is this a good approach or am I missing something completely?
This piece of code recursively delete all the empty folders or directory:
public class DeleteEmptyDir {
private static final String FOLDER_LOCATION = "E:\\TEST";
private static boolean isFinished = false;
public static void main(String[] args) {
do {
isFinished = true;
replaceText(FOLDER_LOCATION);
} while (!isFinished);
}
private static void replaceText(String fileLocation) {
File folder = new File(fileLocation);
File[] listofFiles = folder.listFiles();
if (listofFiles.length == 0) {
System.out.println("Folder Name :: " + folder.getAbsolutePath() + " is deleted.");
folder.delete();
isFinished = false;
} else {
for (int j = 0; j < listofFiles.length; j++) {
File file = listofFiles[j];
if (file.isDirectory()) {
replaceText(file.getAbsolutePath());
}
}
}
}
}
You can use code to delete empty folders using Java.
public static long deleteFolder(String dir) {
File f = new File(dir);
String listFiles[] = f.list();
long totalSize = 0;
for (String file : listFiles) {
File folder = new File(dir + "/" + file);
if (folder.isDirectory()) {
totalSize += deleteFolder(folder.getAbsolutePath());
} else {
totalSize += folder.length();
}
}
if (totalSize ==0) {
f.delete();
}
return totalSize;
}
Shortest code I could come up with is following Java >=8 code:
Files.walk(Paths.get("/my/base/dir/"))
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.filter(File::isDirectory)
.forEach(File::delete);
Add a second (or more) filter statement with whatever clause you need to include/exclude certain folders. File::delete should not delete folders with contents.
Use at own risk.
Kotlin:
fun deleteAllEmptyDirectories(rootPath: Path): Collection<Path> =
mutableListOf<Path>()
.apply {
Files.walk(testPath)
.sorted { p1, p2 -> p2.count() - p1.count() }
.map { it.toFile() }
.filter { it.isDirectory }
.forEach {
if (it.listFiles().all { el -> el.isDirectory && contains(el.toPath()) }) {
val path = it.toPath()
add(path)
Files.delete(path)
}
}
}
Test:
private val testPath = Path.of("build", javaClass.simpleName, UUID.randomUUID().toString())
#Test
fun test() {
Files.createDirectory(testPath)
val dirWithTwoEmptySubdirs = Files.createDirectory(testPath.resolve("dirWithTwoEmptySubdirs"))
val dir1 = Files.createDirectory(dirWithTwoEmptySubdirs.resolve("dir1"))
val dir2 = Files.createDirectory(dirWithTwoEmptySubdirs.resolve("dir2"))
val dirWithOneDiffDir = Files.createDirectory(testPath.resolve("dirWithOneDiffDir"))
var emptyDir = Files.createDirectory(dirWithOneDiffDir.resolve("empty"))
val notEmptyDir = Files.createDirectory(dirWithOneDiffDir.resolve("notempty"))
Files.writeString(notEmptyDir.resolve("file.txt"), "asdf")
assertEquals(
setOf<Path>(dirWithTwoEmptySubdirs, dir1, dir2, emptyDir),
deleteAllEmptyDirectories(testPath).toSet()
)
}
After reading all answers and concluding that all of them have at least one problem I still had to write it myself:
import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public class Temp {
public static void main(String[] args) {
final String path = "D:\\";
deleteEmpty(new File(path));
}
private static int deleteEmpty(File file) {
List<File> toBeDeleted = Arrays.stream(file.listFiles()).sorted() //
.filter(File::isDirectory) //
.filter(f -> f.listFiles().length == deleteEmpty(f)) //
.collect(Collectors.toList());
int size = toBeDeleted.size();
toBeDeleted.forEach(t -> {
final String path = t.getAbsolutePath();
final boolean delete = t.delete();
System.out.println("Deleting: \t" + delete + "\t" + path);
});
return size;
}
}