将多个小文件合并成大文件

有多个按时间戳（20177041735，20177041740）排序下来的文件，需要对这些文件按天（20170417）合并：

/*
* 按天合并文件
*/
public class MergerDayFile {
private static final Log LOG = LogFactory.getLog(MergerDayFile.class);

public static void main(String[] args) throws Exception { Properties properties &#61; new Properties(); // 读取配置文件&#34;./config/mergerFile.properties&#34; FileInputStream in &#61; new FileInputStream(&#34;./config/mergerFile.properties&#34;); properties.load(in); String dest &#61; properties.getProperty(&#34;dest&#34;); String src1 &#61; properties.getProperty(&#34;src1&#34;); String url &#61; properties.getProperty(&#34;url&#34;); // 相同文件按天合并&#xff0c;合并到指定位置 mergerfile(dest, src1, url); try { in.close(); } catch (Exception e) { // TODO: handle exception } } private static void mergerfile(String destPath, String src1, String url) throws Exception { Configuration conf &#61; new Configuration(); // HashMap&lt;String, List&lt;Path&gt;&gt; map &#61; new HashMap&lt;String, List&lt;Path&gt;&gt;(); FileSystem fs &#61; FileSystem.get(new URI(url), conf, &#34;root&#34;); FileStatus[] allFiles &#61; fs.listStatus(new Path(src1)); String lastFile &#61; null; String currentFile &#61; null; List&lt;Path&gt; mergerDay &#61; new ArrayList&lt;Path&gt;(); for (int i &#61; 0; i &lt; allFiles.length; i&#43;&#43;) { FileStatus file &#61; allFiles[i]; Path path &#61; file.getPath(); String name &#61; path.getName(); // 解析文件名 currentFile &#61; parseTime(name); // 解析上一次文件名 String lastFileName &#61; parseTime(lastFile); // 第一次结果判断 if (lastFile &#61;&#61; null) { mergerDay.add(path); lastFile &#61; currentFile; } else if (i &#61;&#61; allFiles.length - 1) { mergerData(destPath, mergerDay, fs); mergerDay.clear(); mergerDay.add(path); mergerData(destPath, mergerDay, fs); } else if (currentFile.equals(lastFileName)) { mergerDay.add(path); } else { mergerData(destPath, mergerDay, fs); mergerDay.clear(); mergerDay.add(path); lastFile &#61; currentFile; } } fs.close(); } private static void mergerData(String destPath, List&lt;Path&gt; mergerDay, FileSystem fs) { Path key &#61; mergerDay.get(0); String name &#61; key.getName(); try { FSDataOutputStream out &#61; fs.create(new Path(destPath &#43; name)); for (Path path : mergerDay) { FSDataInputStream open &#61; fs.open(path); IOUtils.copyBytes(open, out, 8012, false); open.close(); } LOG.info(name &#43; &#34;合并成功&#34;); out.close(); } catch (IOException e) { LOG.error(name &#43; &#34;合并失败&#34;, e); e.printStackTrace(); } } // 按天合并文件 private static String parseTime(String name) { String day &#61; null; try { String[] datas &#61; name.split(&#34;\\.&#34;, 2); String date &#61; datas[0]; day &#61; date.substring(0, 8); } catch (Exception e) { } return day; }

public static void main(String[] args) throws Exception { Properties properties = new Properties(); // 读取配置文件"./config/mergerFile.properties" FileInputStream in = new FileInputStream("./config/mergerFile.properties"); properties.load(in); String dest = properties.getProperty("dest"); String src1 = properties.getProperty("src1"); String url = properties.getProperty("url"); // 相同文件按天合并，合并到指定位置 mergerfile(dest, src1, url); try { in.close(); } catch (Exception e) { // TODO: handle exception } } private static void mergerfile(String destPath, String src1, String url) throws Exception { Configuration conf = new Configuration(); // HashMap<String, List<Path>> map = new HashMap<String, List<Path>>(); FileSystem fs = FileSystem.get(new URI(url), conf, "root"); FileStatus[] allFiles = fs.listStatus(new Path(src1)); String lastFile = null; String currentFile = null; List<Path> mergerDay = new ArrayList<Path>(); for (int i = 0; i < allFiles.length; i++) { FileStatus file = allFiles[i]; Path path = file.getPath(); String name = path.getName(); // 解析文件名 currentFile = parseTime(name); // 解析上一次文件名 String lastFileName = parseTime(lastFile); // 第一次结果判断 if (lastFile == null) { mergerDay.add(path); lastFile = currentFile; } else if (i == allFiles.length - 1) { mergerData(destPath, mergerDay, fs); mergerDay.clear(); mergerDay.add(path); mergerData(destPath, mergerDay, fs); } else if (currentFile.equals(lastFileName)) { mergerDay.add(path); } else { mergerData(destPath, mergerDay, fs); mergerDay.clear(); mergerDay.add(path); lastFile = currentFile; } } fs.close(); } private static void mergerData(String destPath, List<Path> mergerDay, FileSystem fs) { Path key = mergerDay.get(0); String name = key.getName(); try { FSDataOutputStream out = fs.create(new Path(destPath + name)); for (Path path : mergerDay) { FSDataInputStream open = fs.open(path); IOUtils.copyBytes(open, out, 8012, false); open.close(); } LOG.info(name + "合并成功"); out.close(); } catch (IOException e) { LOG.error(name + "合并失败", e); e.printStackTrace(); } } // 按天合并文件 private static String parseTime(String name) { String day = null; try { String[] datas = name.split("\\.", 2); String date = datas[0]; day = date.substring(0, 8); } catch (Exception e) { } return day; }

软粉网

将多个小文件合并成大文件

作者: HUI

发表评论取消回复

给这篇文章的作者打赏

作者: HUI

相关文章

java元注解及源码浅析

【MySQL语句】MySQL基础语句，应付期末考试，木得问题~

Django–初识

CentOS7关闭防火墙

Spring框架学习记录

SpringBoot核心配置与注解02

发表评论 取消回复

发表评论取消回复