\documentclass[11pt,twoside]{article}\makeatletter

\IfFileExists{xcolor.sty}%
  {\RequirePackage{xcolor}}%
  {\RequirePackage{color}}
\usepackage{colortbl}
\usepackage{wrapfig}
\usepackage{ifxetex}
\ifxetex
  \usepackage{fontspec}
  \usepackage{xunicode}
  \catcode`⃥=\active \def⃥{\textbackslash}
  \catcode`❴=\active \def❴{\{}
  \catcode`❵=\active \def❵{\}}
  \def\textJapanese{\fontspec{Noto Sans CJK JP}}
  \def\textChinese{\fontspec{Noto Sans CJK SC}}
  \def\textKorean{\fontspec{Noto Sans CJK KR}}
  \setmonofont{DejaVu Sans Mono}
  
\else
  \IfFileExists{utf8x.def}%
   {\usepackage[utf8x]{inputenc}
      \PrerenderUnicode{–}
    }%
   {\usepackage[utf8]{inputenc}}
  \usepackage[english]{babel}
  \usepackage[T1]{fontenc}
  \usepackage{float}
  \usepackage[]{ucs}
  \uc@dclc{8421}{default}{\textbackslash }
  \uc@dclc{10100}{default}{\{}
  \uc@dclc{10101}{default}{\}}
  \uc@dclc{8491}{default}{\AA{}}
  \uc@dclc{8239}{default}{\,}
  \uc@dclc{20154}{default}{ }
  \uc@dclc{10148}{default}{>}
  \def\textschwa{\rotatebox{-90}{e}}
  \def\textJapanese{}
  \def\textChinese{}
  \IfFileExists{tipa.sty}{\usepackage{tipa}}{}
\fi
\def\exampleFont{\ttfamily\small}
\DeclareTextSymbol{\textpi}{OML}{25}
\usepackage{relsize}
\RequirePackage{array}
\def\@testpach{\@chclass
 \ifnum \@lastchclass=6 \@ne \@chnum \@ne \else
  \ifnum \@lastchclass=7 5 \else
   \ifnum \@lastchclass=8 \tw@ \else
    \ifnum \@lastchclass=9 \thr@@
   \else \z@
   \ifnum \@lastchclass = 10 \else
   \edef\@nextchar{\expandafter\string\@nextchar}%
   \@chnum
   \if \@nextchar c\z@ \else
    \if \@nextchar l\@ne \else
     \if \@nextchar r\tw@ \else
   \z@ \@chclass
   \if\@nextchar |\@ne \else
    \if \@nextchar !6 \else
     \if \@nextchar @7 \else
      \if \@nextchar (8 \else
       \if \@nextchar )9 \else
  10
  \@chnum
  \if \@nextchar m\thr@@\else
   \if \@nextchar p4 \else
    \if \@nextchar b5 \else
   \z@ \@chclass \z@ \@preamerr \z@ \fi \fi \fi \fi
   \fi \fi  \fi  \fi  \fi  \fi  \fi \fi \fi \fi \fi \fi}
\gdef\arraybackslash{\let\\=\@arraycr}
\def\@textsubscript#1{{\m@th\ensuremath{_{\mbox{\fontsize\sf@size\z@#1}}}}}
\def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}}
\def\abbr{}
\def\corr{}
\def\expan{}
\def\gap{}
\def\orig{}
\def\reg{}
\def\ref{}
\def\sic{}
\def\persName{}\def\name{}
\def\placeName{}
\def\orgName{}
\def\textcal#1{{\fontspec{Lucida Calligraphy}#1}}
\def\textgothic#1{{\fontspec{Lucida Blackletter}#1}}
\def\textlarge#1{{\large #1}}
\def\textoverbar#1{\ensuremath{\overline{#1}}}
\def\textquoted#1{‘#1’}
\def\textsmall#1{{\small #1}}
\def\textsubscript#1{\@textsubscript{\selectfont#1}}
\def\textxi{\ensuremath{\xi}}
\def\titlem{\itshape}
\newenvironment{biblfree}{}{\ifvmode\par\fi }
\newenvironment{bibl}{}{}
\newenvironment{byline}{\vskip6pt\itshape\fontsize{16pt}{18pt}\selectfont}{\par }
\newenvironment{citbibl}{}{\ifvmode\par\fi }
\newenvironment{docAuthor}{\ifvmode\vskip4pt\fontsize{16pt}{18pt}\selectfont\fi\itshape}{\ifvmode\par\fi }
\newenvironment{docDate}{}{\ifvmode\par\fi }
\newenvironment{docImprint}{\vskip 6pt}{\ifvmode\par\fi }
\newenvironment{docTitle}{\vskip6pt\bfseries\fontsize{22pt}{25pt}\selectfont}{\par }
\newenvironment{msHead}{\vskip 6pt}{\par}
\newenvironment{msItem}{\vskip 6pt}{\par}
\newenvironment{rubric}{}{}
\newenvironment{titlePart}{}{\par }

\newcolumntype{L}[1]{){\raggedright\arraybackslash}p{#1}}
\newcolumntype{C}[1]{){\centering\arraybackslash}p{#1}}
\newcolumntype{R}[1]{){\raggedleft\arraybackslash}p{#1}}
\newcolumntype{P}[1]{){\arraybackslash}p{#1}}
\newcolumntype{B}[1]{){\arraybackslash}b{#1}}
\newcolumntype{M}[1]{){\arraybackslash}m{#1}}
\definecolor{label}{gray}{0.75}
\def\unusedattribute#1{\sout{\textcolor{label}{#1}}}
\DeclareRobustCommand*{\xref}{\hyper@normalise\xref@}
\def\xref@#1#2{\hyper@linkurl{#2}{#1}}
\begingroup
\catcode`\_=\active
\gdef_#1{\ensuremath{\sb{\mathrm{#1}}}}
\endgroup
\mathcode`\_=\string"8000
\catcode`\_=12\relax

\usepackage[a4paper,twoside,lmargin=1in,rmargin=1in,tmargin=1in,bmargin=1in,marginparwidth=0.75in]{geometry}
\usepackage{framed}

\definecolor{shadecolor}{gray}{0.95}
\usepackage{longtable}
\usepackage[normalem]{ulem}
\usepackage{fancyvrb}
\usepackage{fancyhdr}
\usepackage{graphicx}
\usepackage{marginnote}

\renewcommand{\@cite}[1]{#1}


\renewcommand*{\marginfont}{\itshape\footnotesize}

\def\Gin@extensions{.pdf,.png,.jpg,.mps,.tif}

  \pagestyle{fancy}

\usepackage[pdftitle={Critical Analysis of Solutions to Hadoop Small File Problem},
 pdfauthor={}]{hyperref}
\hyperbaseurl{}

	 \paperwidth210mm
	 \paperheight297mm
              
\def\@pnumwidth{1.55em}
\def\@tocrmarg {2.55em}
\def\@dotsep{4.5}
\setcounter{tocdepth}{3}
\clubpenalty=8000
\emergencystretch 3em
\hbadness=4000
\hyphenpenalty=400
\pretolerance=750
\tolerance=2000
\vbadness=4000
\widowpenalty=10000

\renewcommand\section{\@startsection {section}{1}{\z@}%
     {-1.75ex \@plus -0.5ex \@minus -.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\Large\bfseries}}
\renewcommand\subsection{\@startsection{subsection}{2}{\z@}%
     {-1.75ex\@plus -0.5ex \@minus- .2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\Large}}
\renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}%
     {-1.5ex\@plus -0.35ex \@minus -.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\large}}
\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}%
     {-1ex \@plus-0.35ex \@minus -0.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\normalsize}}
\renewcommand\subparagraph{\@startsection{subparagraph}{5}{\parindent}%
     {1.5ex \@plus1ex \@minus .2ex}%
     {-1em}%
     {\reset@font\normalsize\bfseries}}


\def\l@section#1#2{\addpenalty{\@secpenalty} \addvspace{1.0em plus 1pt}
 \@tempdima 1.5em \begingroup
 \parindent \z@ \rightskip \@pnumwidth 
 \parfillskip -\@pnumwidth 
 \bfseries \leavevmode #1\hfil \hbox to\@pnumwidth{\hss #2}\par
 \endgroup}
\def\l@subsection{\@dottedtocline{2}{1.5em}{2.3em}}
\def\l@subsubsection{\@dottedtocline{3}{3.8em}{3.2em}}
\def\l@paragraph{\@dottedtocline{4}{7.0em}{4.1em}}
\def\l@subparagraph{\@dottedtocline{5}{10em}{5em}}
\@ifundefined{c@section}{\newcounter{section}}{}
\@ifundefined{c@chapter}{\newcounter{chapter}}{}
\newif\if@mainmatter 
\@mainmattertrue
\def\chaptername{Chapter}
\def\frontmatter{%
  \pagenumbering{roman}
  \def\thechapter{\@roman\c@chapter}
  \def\theHchapter{\roman{chapter}}
  \def\thesection{\@roman\c@section}
  \def\theHsection{\roman{section}}
  \def\@chapapp{}%
}
\def\mainmatter{%
  \cleardoublepage
  \def\thechapter{\@arabic\c@chapter}
  \setcounter{chapter}{0}
  \setcounter{section}{0}
  \pagenumbering{arabic}
  \setcounter{secnumdepth}{6}
  \def\@chapapp{\chaptername}%
  \def\theHchapter{\arabic{chapter}}
  \def\thesection{\@arabic\c@section}
  \def\theHsection{\arabic{section}}
}
\def\backmatter{%
  \cleardoublepage
  \setcounter{chapter}{0}
  \setcounter{section}{0}
  \setcounter{secnumdepth}{2}
  \def\@chapapp{\appendixname}%
  \def\thechapter{\@Alph\c@chapter}
  \def\theHchapter{\Alph{chapter}}
  \appendix
}
\newenvironment{bibitemlist}[1]{%
   \list{\@biblabel{\@arabic\c@enumiv}}%
       {\settowidth\labelwidth{\@biblabel{#1}}%
        \leftmargin\labelwidth
        \advance\leftmargin\labelsep
        \@openbib@code
        \usecounter{enumiv}%
        \let\p@enumiv\@empty
        \renewcommand\theenumiv{\@arabic\c@enumiv}%
	}%
  \sloppy
  \clubpenalty4000
  \@clubpenalty \clubpenalty
  \widowpenalty4000%
  \sfcode`\.\@m}%
  {\def\@noitemerr
    {\@latex@warning{Empty `bibitemlist' environment}}%
    \endlist}

\def\tableofcontents{\section*{\contentsname}\@starttoc{toc}}
\parskip0pt
\parindent1em
\def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}}
\newenvironment{reflist}{%
  \begin{raggedright}\begin{list}{}
  {%
   \setlength{\topsep}{0pt}%
   \setlength{\rightmargin}{0.25in}%
   \setlength{\itemsep}{0pt}%
   \setlength{\itemindent}{0pt}%
   \setlength{\parskip}{0pt}%
   \setlength{\parsep}{2pt}%
   \def\makelabel##1{\itshape ##1}}%
  }
  {\end{list}\end{raggedright}}
\newenvironment{sansreflist}{%
  \begin{raggedright}\begin{list}{}
  {%
   \setlength{\topsep}{0pt}%
   \setlength{\rightmargin}{0.25in}%
   \setlength{\itemindent}{0pt}%
   \setlength{\parskip}{0pt}%
   \setlength{\itemsep}{0pt}%
   \setlength{\parsep}{2pt}%
   \def\makelabel##1{\upshape ##1}}%
  }
  {\end{list}\end{raggedright}}
\newenvironment{specHead}[2]%
 {\vspace{20pt}\hrule\vspace{10pt}%
  \phantomsection\label{#1}\markright{#2}%

  \pdfbookmark[2]{#2}{#1}%
  \hspace{-0.75in}{\bfseries\fontsize{16pt}{18pt}\selectfont#2}%
  }{}
      \def\TheFullDate{2023 1970-01-01 (revised: 28 Year 2023 01 January 1970)}
\def\TheID{\makeatother }
\def\TheDate{2023 1970-01-01}
\title{Critical Analysis of Solutions to Hadoop Small File Problem}
\author{}\makeatletter 
\makeatletter
\newcommand*{\cleartoleftpage}{%
  \clearpage
    \if@twoside
    \ifodd\c@page
      \hbox{}\newpage
      \if@twocolumn
        \hbox{}\newpage
      \fi
    \fi
  \fi
}
\makeatother
\makeatletter
\thispagestyle{empty}
\markright{\@title}\markboth{\@title}{\@author}
\renewcommand\small{\@setfontsize\small{9pt}{11pt}\abovedisplayskip 8.5\p@ plus3\p@ minus4\p@
\belowdisplayskip \abovedisplayskip
\abovedisplayshortskip \z@ plus2\p@
\belowdisplayshortskip 4\p@ plus2\p@ minus2\p@
\def\@listi{\leftmargin\leftmargini
               \topsep 2\p@ plus1\p@ minus1\p@
               \parsep 2\p@ plus\p@ minus\p@
               \itemsep 1pt}
}
\makeatother
\fvset{frame=single,numberblanklines=false,xleftmargin=5mm,xrightmargin=5mm}
\fancyhf{} 
\setlength{\headheight}{14pt}
\fancyhead[LE]{\bfseries\leftmark} 
\fancyhead[RO]{\bfseries\rightmark} 
\fancyfoot[RO]{}
\fancyfoot[CO]{\thepage}
\fancyfoot[LO]{\TheID}
\fancyfoot[LE]{}
\fancyfoot[CE]{\thepage}
\fancyfoot[RE]{\TheID}
\hypersetup{citebordercolor=0.75 0.75 0.75,linkbordercolor=0.75 0.75 0.75,urlbordercolor=0.75 0.75 0.75,bookmarksnumbered=true}
\fancypagestyle{plain}{\fancyhead{}\renewcommand{\headrulewidth}{0pt}}

\date{}
\usepackage{authblk}

\providecommand{\keywords}[1]
{
\footnotesize
  \textbf{\textit{Index terms---}} #1
}

\usepackage{graphicx,xcolor}
\definecolor{GJBlue}{HTML}{273B81}
\definecolor{GJLightBlue}{HTML}{0A9DD9}
\definecolor{GJMediumGrey}{HTML}{6D6E70}
\definecolor{GJLightGrey}{HTML}{929497} 

\renewenvironment{abstract}{%
   \setlength{\parindent}{0pt}\raggedright
   \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}}
   \vskip16pt
   \textcolor{GJBlue}{\large\bfseries\abstractname\space}
}{%   
   \vskip8pt
   \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}}
   \vskip16pt
}

\usepackage[absolute,overlay]{textpos}

\makeatother 
      \usepackage{lineno}
      \linenumbers
      
\begin{document}

             \author[1]{Prof. hwetha K  S}

             \author[2]{Dr. Chandramouli  H}

\renewcommand\Authands{ and }

\date{\small \em Received: 1 January 1970 Accepted: 1 January 1970 Published: 1 January 1970}

\maketitle


\begin{abstract}
        


Hadoop big data platform is designed to process large volume of data. Small file problem is a performance bottleneck in Hadoop processing. Small files lower than the block size of Hadoop creates huge storage overhead at Namenode?s and also wastes computational resources due to spawning of many map tasks. Various solutions like merging small files, mapping multiple map threads to same java virtual machine instance etc have been proposed to solve the small file problems in Hadoop. This survey does a critical analysis of existing works addressing small file problems in Hadoop and its variant platforms like Spark. The aim is to understand their effectiveness in reducing the storage/computational overhead and identify the open issues for further research.

\end{abstract}


\keywords{}

\begin{textblock*}{18cm}(1cm,1cm) % {block width} (coords) 
\textcolor{GJBlue}{\LARGE Global Journals \LaTeX\ JournalKaleidoscope\texttrademark}
\end{textblock*}

\begin{textblock*}{18cm}(1.4cm,1.5cm) % {block width} (coords) 
\textcolor{GJBlue}{\footnotesize \\ Artificial Intelligence formulated this projection for compatibility purposes from the original article published at Global Journals. However, this technology is currently in beta. \emph{Therefore, kindly ignore odd layouts, missed formulae, text, tables, or figures.}}
\end{textblock*}


\begin{textblock*}{10cm}(1.05cm,3cm)
{{\textit{CrossRef DOI of original article:}} \underline{}}
\end{textblock*}\let\tabcellsep& 	 	 		 
\section[{I. Introduction}]{I. Introduction}\par
adoop is an open source big data processing platform designed to process large volume of data. The data is kept in form of files in Hadoop distributed file system (HDFS). A map job is spawned on a java virtual machine (JVM) instance for each file in HDFS. The file data is copied to a memory block and the block is passed to map task. In addition, a object instance is created for each file in the Namenode of Hadoop to facilitate processing. When the file size is more than or equal to block size, maximum performance gain in achieved in terms of number of maps spawned and the meta data storage overhead at Namenode. In case of IoT applications, the data files are small (less than 2KB) and when these files are stored in HDFS for data processing, it affects the Hadoop performance \hyperref[b0]{[1]}\hyperref[b1]{[2]}. On one hand, it drastically increases the storage overhead at Namenode for object bookkeeping \hyperref[b2]{[3]}. On another hand it exhausts the computational resources by spawning multiple map tasks which only lasts for smaller duration to process small files. The time spent in bootstrapping the map task becomes higher than data processing time in case of small files. Various solutions have been proposed addressing the Hadoop small file problem. The existing solutions can be categorized as: (i) file merging solutions, (ii) file caching solutions, (iii) optimizing Hadoop cluster structure and (iv) Map task optimizations. In file merging solutions, pre-treatment of small files is done to form a big file and this big file is stored in HDFS. In file caching solutions, files are sent to a file queue, and when queue size crosses threshold files are sent to processing in a systematic manner. In Hadoop cluster structure optimization solutions, hierarchical memory structure is created combining cache and HDFS memory to reduce the overhead due to single name node. In map task optimization solution, number of JVM instances spawned for map tasks are reduced and shared.\par
This work does a critical analysis on various solutions in the above four categories of file merging, file caching, Hadoop cluster structure optimization and map task optimization. The effectiveness of each of the solutions in terms of storage and computation are analyzed and their open issues are identified. Based on the open issues, a prospective solution framework is designed and detailed. 
\section[{II. Survey}]{II. Survey}\par
Ahad et al \hyperref[b3]{[4]} proposed a dynamic merging strategy based on the file type for Hadoop. Dynamic variable size portioning is applied to blocks and the file contents are fitted to blocks using next fit allocation policy. By this way large file is created and saved to HDFS. In addition, authors also secured the block using Twofish cryptographic technique. The solution reduced name node memory, number of data blocks and processing time. Merging was done only based on file types without considering the context and their semantic relation. Siddiqui et al \hyperref[b4]{[5]} proposed a cache based block management technique for Hadoop as a replacement for default Hadoop Archives (HAR). A logical chain of small files is built and transferred to data blocks. In addition, efficient read/write on blocks was facilitated using block manager. Though the solution achieved more than 92\% space utilization of data blocks, small files are merged only based on size, without considering the semantic relations and content characteristics. Zhai et al \hyperref[b5]{[6]} built a index based archive file to solve the small file problem in Hadoop. The small files are merged to large file and metadata record is created to retrieve each file content. Meta data records are arranged into buckets. An order preserving hash is created over metadata records. The hash and the metadata records are in turn written to a index file. The index files helps to retrieve the file contents for processing. This method is able to save atleast 11\% disk space but the solution access efficiency becomes lower with large number of small files. Also the indexing does not support streaming inputs. Cai et al \hyperref[b6]{[7]} proposed a file merging algorithm based on two factors of distribution of the files and the correlation of the file. Correlation between files is built based on their history of access and the highly correlated files are kept in the same block. Through experiments, author found that placing highly correlated files in same block improved the speed up. The correlation is not based on content characteristics so over a period of time, performance can reduce. Choi et al \hyperref[b7]{[8]} integrated combinedfileinputformat and JVM reuse to solve the small file problem. Small files are combined till block size and passed to map task. JVM instances are reused for the map task , so they overhead of JVM bootstrap is minimized. Though the integration reduces the computational overhead, the approach combined files in order without considering their semantics. Also the memory buildup due to JVM reuse can crash the tasks due to inefficient memory management. Peng et al \hyperref[b8]{[9]} combined merging and caching techniques to solve the small file problem. User based collaborative filtering is applied to learn the correlation between the files. Files with higher correlation are merged into single large file. Remote procedure call (RPC) requests to fetch the block information about the files are reduced by caching the access requests and looking into cache for the blocks before placing RPC requests. By this way, authors were able to reduce the file access time by 50\% and increase storage utilization by 25\% compared to default Hadoop. The scheme does not works well for streaming data, as the correlation model proposed in this work is not adaptive to streaming data. Niazi et al \hyperref[b9]{[10]} proposed a new technique called inode stuffing to solve the small file problem. For small files, the metadata and data block are combined and decoupling is maintained only for large files. The approach is not scalable as it increases the metadata storage overhead at Namenodes. Jing et al \hyperref[b10]{[11]} proposed a dynamic queue method to solve the small file problem. The files are first classified using the period classification algorithm. The algorithm calculates similarity score based on sentence similarity between two documents. The similar files are then merged to large file using multiple queues for specific file sizes. Authors also used file pre-fetching strategy to improve the efficiency of file access. Analyzing similarity between pairs is a cumbersome task for large number of files. Sharma et al \hyperref[b11]{[12]} proposed a dual merge technique called Hash Based-Extended Hadoop Archive to solve the small file problem in Hadoop. The small files are merged using two level compaction. This reduces the storage overhead at Namenode and increase the data block space utilization at Datanodes. File access is made efficient using two level hash function. The proposed solution is atleast 13\% faster compared to default Hadoop. The files were merged without considering the content characteristics and their semantics. Wang et al \hyperref[b12]{[13]} combined merging and caching to solve the small file problem in Hadoop. Authors proposed a equilibrium merger queue algorithm to merge small files to Hadoop block size and then merged file is saved to HDFS. Indexing is built to access small files. To reduce the communication overhead between the client and Namenode for small file access, pre-fetched cache is used. With the cache, the number of RPC calls to name node is reduced. The memory consumption at Namenode drastically reduced in the proposed solution compared to default Hadoop Archives. Contents were merged without considering their content characteristics and semantic correlation. Ali et al \hyperref[b13]{[14]} proposed a enhanced best fit merging algorithm to merge small files based on type and size. The merging is done till Hadoop block size is reached and merged file is saved to HDFS. Author found that merging improved Hadoop storage utilization by 64\% but the file access time was higher in this work. Prasanna et al \hyperref[b14]{[15]} compressed many small files into a zip file to the size of Hadoop data block and saved to disk. This increased the disk utilization of data nodes and name nodes. But the computational overhead in compressing stage and decompressing during processing is higher. Huang et al \hyperref[b15]{[16]} addressed the small file problem for the case of images in Hadoop. A two level model was proposed specific to medical images. The images were grouped at first level based on series and next level based on examination. The grouped images are saved to data blocks in HDFS. Indexing and pre-fetching is done to done is reduce the access time for small image files. The pre-fetching algorithm did not have higher cache hit. Renner et al \hyperref[b16]{[17]} extended the Hadoop archive to appendable file format to solve the small file problem. Small files are appended to existing archive data files whose block size is not completely used. Authors used first fit algorithm to select the data blocks. In addition indexing is done to facilitate faster access. Red black tree structure is used for indexing for efficient lookup. Though this scheme improved the data block utilization, appending is done without considering content characteristics and semantic similarity. Liu et al \hyperref[b17]{[18]} proposed a file merging strategy based on content similarity. Files are converted to vector space features and correlation between the features is measured using cosine similarity. When cosine similarity is greater than threshold, files are merged. In addition authors used pre-fetching and caching to speed up the file access. Constructing a global feature space for streaming data is difficult and thus this approach is not suitable for streaming data.Lyu et al \hyperref[b18]{[19]} proposed an optimized merging strategy to solve small file problem. The small files are merged based on size in such that way block size is fully utilized. In addition authors used pre-fetching and caching to increase the access speed. Only block size utilization was considered as the only criteria for merging without considering content characteristics and semantic relations. Similar to it Mu et al \hyperref[b19]{[20]} proposed an optimization strategy to maximally fill the existing Hadoop archive by appending small files. In addition author also used secondary index to speed up the execution of file access. But here too merging was done without considering content characteristics and semantic relation. Wang et al \hyperref[b20]{[21]} used probabilistic latent semantic analysis to determine the user access pattern and based on it small files are merged to a large file and placed in HDFS. In addition author also improved the pre-fetching hit ratio based user access transition pattern. Both the strategies improvised the speed of access and data block utilization. But this scheme is not suitable for multi user environment as for each user, a merging order must be kept and this increases the storage overhead. He et al \hyperref[b21]{[22]} merging the small files based on balance of data blocks. The aim was to increase the data block utilization. Merging did not consider content characteristics and their semantic relation. Fu et al \hyperref[b22]{[23]} proposed an flat storage architecture to handle the small files. In this scheme, both files and meta data are collocated with meta size fixed for any number of small files. This is facilitates by meta data having only pointer to related information in its index. But the scheme is not suited for Hadoop as collocation causes higher access overhead for large files. Tao et al \hyperref[b23]{[24]} merged small files to large file and built a linear hash to small files to speed up access. File size was the only criteria considered for merging. Bok et al \hyperref[b24]{[25]} integrated file merging and caching to solve the small file problem. Author used two level of cache for small files, so that access requests to - Cai et al \hyperref[b6]{[7]} file merging algorithm based on two factors of distribution of the files and the correlation of the file The correlation is not based on content characteristics Choi et al \hyperref[b7]{[8]} integrated combinedfileinputformat and JVM reuse to solve the small file problem memory buildup due to JVM reuse can crash the tasks due to inefficient memory management Peng et al \hyperref[b8]{[9]} combined merging and caching techniques to solve the small file problem\par
The scheme does not works well for streaming data, as the correlation model proposed in this work is not adaptive to streaming data Niazi et al \hyperref[b9]{[10]} Coupling both meta data and small file together.\par
The approach is not scalable as it increases the metadata storage overhead at Namenodes Jing et al \hyperref[b10]{[11]} Files classified using the period classification algorithm and merged based on similarity Analyzing similarity between pairs is a cumbersome task for large number of files Sharma et al \hyperref[b11]{[12]} Hash Based-Extended Hadoop Archive to solve the small file problem\par
The files were merged without considering the content characteristics and their semantics.\par
Wang et al \hyperref[b12]{[13]} combined merging and caching to solve the small file problem Contents were merged without considering their content characteristics and semantic correlation Ali et al \hyperref[b13]{[14]} enhanced best fit merging algorithm to merge small files based on type and size. file access time was higher in this work\par
Huang et al \hyperref[b15]{[16]} A two level model was proposed specific to medical images\par
The pre-fetching algorithm did not have higher cache hit Renner et al \hyperref[b16]{[17]} Small files are appended to existing archive data files Appending is done without considering content characteristics and semantic similarity Liu et al \hyperref[b17]{[18]} File content based merging Constructing a global feature space for streaming data is difficult and thus this approach is not suitable for streaming data Lyu et al \hyperref[b18]{[19]} optimized merging strategy to solve small file problem.\par
Only block size utilization was considered as the only criteria for merging without considering content characteristics and semantic relations Wang et al \hyperref[b20]{[21]} probabilistic latent semantic analysis to determine the user access pattern and based on it small files are merged to a large file scheme is not suitable for multi user environment as for each user, a merging order must be kept and this increases the storage overhead He et al \hyperref[b21]{[22]} merging the small files based on balance of data blocks\par
Merging did not consider content characteristics and their semantic relation Fu et al \hyperref[b22]{[23]} flat storage architecture collocating metadata and file in same object the scheme is not suited for Hadoop as collocation causes higher access overhead for large files Tao et al \hyperref[b23]{[24]} merged small files to large file and built a linear hash to small files to speed up access File size was the only criteria considered for merging Bok et al \hyperref[b24]{[25]} integrated file merging and caching to solve the small file problem\par
The merging was based only on size without considering the content characteristics and semantic similarity Caching on global context can provide better performance for some users and can give worst performance for other users. To solve this access time discrepancy among the users, personalized caching strategy must be employed.\par
Steaming Support: Most of the merging schemes does not handle the steaming data effectively. Streaming data content similarity cannot be computed effectively using vector space modeling and their merging can become ineffective. Merging based on streaming arrival patterns has not been considered in earlier works. 
\section[{IV. Research Directions}]{IV. Research Directions}\par
Based on the open issues identified, a prospective framework for further research is presented in Figure  {\ref 1}.\par
The framework addresses three problem areas of context specific merging, personalized access and streaming support.\par
Context Specific Merging: It can be facilitated and made adaptive using machine learning. Based on the application contexts and inherent data characteristics the files to be merged can be found. Blocks can be categorized based on context and small files can be categorized based on context. Context based merging is the realized to merge files and blocks based on context similarity. Instead of flat context, hierarchical context can be learnt automatically from file summarization. File summarization strategies specific to file types can be proposed to identify the context to be associated with files and blocks.\par
Personalized Access: User can be clustered based on their content access patterns over a temporal duration and multiple caches can be maintained for each user group. Also the cache item management can be based on multi criteria optimization instead of LRU mechanisms. The items to pre-fetch can be identified based on context associated with files. By this way access speed up can be increased and optimized specific to each user group.\par
Streaming Support: To support streaming data, the context must be learnt dynamically in a light weight manner and association of small file to blocks must be done based on context. To learn context in a light weight manner, the streaming data characteristics and their arrival patterns must be used. 
\section[{V. Conclusion}]{V. Conclusion}\par
This survey made a critical analysis of existing solutions for small file problem in Hadoop. The solutions were analyzed in four categories of file merging solutions, file caching solutions, optimizing Hadoop cluster structure and Map task optimizations. Based on the survey, three open issues of context specific merging, personalized access and streaming support are identified. Prospective solutions to these three open issues were identified and a solution roadmap for further exploration in this area was documented.\begin{figure}[htbp]
\noindent\textbf{} \par 
\begin{longtable}{P{0.85\textwidth}}
Critical Analysis of Solutions to Hadoop Small File Problem\\
Year 2023\\
25\\
Volume XXIII Issue II Version I\\
( ) C\\
Global Journal of Computer Science and Technology\\
Figure 1: Research direction framework\\
© 2023 Global Journals\end{longtable} \par
 
\caption{\label{tab_0}Context based Block Categorization Machine learning based Context identification Small files Context based file to block association Context attested Small files Context attested Blocks User profiling Context based cache for user group 1 User access request User clustering Context based cache for user group 2 Context based cache for user group n Context based prefetching multi criteria optimization based caching}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{1} \par 
\begin{longtable}{P{0.09042553191489361\textwidth}P{0.3273404255319149\textwidth}P{0.4322340425531915\textwidth}}
Work\tabcellsep Solution for Small file Problem\tabcellsep Gap\\
\tabcellsep \tabcellsep Merging was done only based on file types\\
Ahad et al [4]\tabcellsep dynamic merging strategy based on the file type\tabcellsep without considering the context and their\\
\tabcellsep \tabcellsep semantic relation\\
\tabcellsep \tabcellsep small files are merged only based on size,\\
Siddiqui et al [5]\tabcellsep cache based block management technique\tabcellsep without considering the semantic relations\\
\tabcellsep \tabcellsep and content characteristics\\
Zhai et al [6]\tabcellsep a index based archive file with order preserving hash for speedup\tabcellsep Does not support streaming\end{longtable} \par
 
\caption{\label{tab_1}Table 1 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{1} \par 
\begin{longtable}{P{0.85\textwidth}}
Critical Analysis of Solutions to Hadoop Small File Problem\\
Year 2023\\
Volume XXIII Issue II Version I\\
( ) C\\
Global Journal of Computer Science and Technology\\
© 2023 Global Journals\end{longtable} \par
 
\caption{\label{tab_2}Table 1 .}\end{figure}
 			\label{foot_0}\footnote{\label{foot_0} © 2023 Global Journals} 		 		\backmatter  			  				\begin{bibitemlist}{1}
\bibitem[Sharma et al. ()]{b11}\label{b11} 	 		‘A Dynamic Repository Approach for Small File Management With Fast Access Time on Hadoop Cluster: Hash Based Extended Hadoop Archive’.  		 			V S Sharma 		,  		 			A Afthanorhan 		,  		 			N C Barwar 		,  		 			S Singh 		,  		 			H Malik 		.  	 	 		\textit{IEEE Access}  		2022. 10 p. .  	 
\bibitem[Renner et al. ()]{b16}\label{b16} 	 		‘Addressing Hadoop's Small File Problem With an Appendable Archive File Format’.  		 			Thomas Renner 		,  		 			Johannes Müller 		,  		 			Lauritz Thamsen 		,  		 			Odej Kao 		.  	 	 		\textit{Proceedings of the Computing Frontiers Conference (CF'17)},  		 (the Computing Frontiers Conference (CF'17)New York, NY, USA)  		2017. Association for Computing Machinery. p. .  	 
\bibitem[Wang et al. (2015)]{b20}\label{b20} 	 		‘An effective strategy for improving small \textunderscore le problem in distributed file system’.  		 			T Wang 		,  		 			S Yao 		,  		 			Z Xu 		,  		 			L Xiong 		,  		 			X Gu 		,  		 			X Yang 		.  	 	 		\textit{Proc. 2nd Int. Conf},  		 (2nd Int. Conf)  		Apr. 2015. p. .  	 
\bibitem[Bok et al. (2017)]{b24}\label{b24} 	 		‘An efficient distributed caching for accessing small files in HDFS’.  		 			K Bok 		,  		 			H Oh 		,  		 			J Lim 		,  		 			Y Pae 		,  		 			H Choi 		,  		 			B Lee 		,  		 			J Yoo 		.  	 	 		\textit{Cluster Comput}  		Dec. 2017. 20  (4)  p. .  	 
\bibitem[Cai et al. ()]{b6}\label{b6} 	 		\textit{An optimization strategy of massive small files storage based on HDFS},  		 			Xun Cai 		,  		 			Chen 		,  		 			\& Cai 		,  		 			Yi Liang 		.  		 \xref{http://dx.doi.org/10.2991/jiaet-18.2018.40}{10.2991/jiaet-18.2018.40}.  		2018.  	 
\bibitem[Dong et al. ()]{b2}\label{b2} 	 		‘An optimized approach for storing and accessing small files on cloud storage’.  		 			Bo Dong 		,  		 			Qinghua Zheng 		,  		 			Feng Tian 		,  		 			Kuo-Ming Chao 		,  		 			Rui Ma 		,  		 			Rachid Anane 		.  	 	 		\textit{Journal of Network and Computer Applications}  		2012. 2012. Elsevier. 35 p. .  	 
\bibitem[Jing et al. ()]{b10}\label{b10} 	 		‘An optimized method of HDFS for massive small files storage’.  		 			Weipeng \& Jing 		,  		 			Tong 		,  		 			Guangsheng \& Danyu \& Chen 		,  		 			Chuanyu Zhao 		,  		 			Liangkuan Zhu 		.  		 \xref{http://dx.doi.org/15.21-21.10.2298/CSIS171015021J}{15.21-21.10.2298/CSIS171015021J}.  	 	 		\textit{Computer Science and Information Systems}  		2018.  	 
\bibitem[Lyu et al. (2017)]{b18}\label{b18} 	 		‘An optimized strategy for small files storing and accessing in HDFS’.  		 			Y Lyu 		,  		 			X Fan 		,  		 			K Liu 		.  	 	 		\textit{Proc. IEEE Int. Conf. CSE, IEEE Int. Conf. EUC},  		 (IEEE Int. Conf. CSE, IEEE Int. Conf. EUC)  		Jul. 2017. p. .  	 
\bibitem[Ahad and Biswas ()]{b3}\label{b3} 	 		‘Dynamic Merging based Small File Storage (DM-SFS) Architecture for Efficiently Storing Small Size Files in Hadoop’.  		 			Mohd Ahad 		,  		 			Ranjit Biswas 		.  		 \xref{http://dx.doi.org/1626-1635.10.1016/j.procs.2018.05.128}{1626-1635.10.1016/j.procs.2018.05.128}.  	 	 		\textit{Procedia Computer Science}  		2018. 132.  	 
\bibitem[Ali et al. ()]{b13}\label{b13} 	 		‘Enhanced best fit algorithm for merging small files’.  		 			A Ali 		,  		 			N M Mirza 		,  		 			M K Ishak 		.  	 	 		\textit{Computer Systems Science and Engineering}  		2023. 46  (1)  p. .  	 
\bibitem[Zhai et al. ()]{b5}\label{b5} 	 		‘Hadoop Perfect File: A fast and memory-efficient metadata access archive file to face small files problem in HDFS’.  		 			Yanlong \& Zhai 		,  		 			Jude \& Tchaye-Kondi 		,  		 			Kwei-Jay \& Lin 		,  		 			Zhu 		,  		 			\& Liehuang 		,  		 			Tao 		,  		 			\& Wenjun 		,  		 			Du 		,  		 			Mohsen Xiaojiang \& Guizani 		.  		 \xref{http://dx.doi.org/156.10.1016/j.jpdc.2021.05.011}{156.10.1016/j.jpdc.2021.05.011}.  	 	 		\textit{Journal of Parallel and Distributed Computing}  		2021.  	 
\bibitem[Huang et al. ()]{b15}\label{b15} 	 		‘Hadoop-Based Medical Image Storage and Access Method for Examination Series’.  		 			Xin Huang 		,  		 			Wenlong Yi 		,  		 			Jiwei Wang 		,  		 			Zhijian Xu 		.  	 	 		\textit{Mathematical Problems in Engineering}  		2021. 2021.  	 	 (Article ID 5525009, 10 pages) 
\bibitem[Choi et al. ()]{b7}\label{b7} 	 		‘Improved performance optimization for massive small files in cloud computing environment’.  		 			C Choi 		,  		 			C Choi 		,  		 			J Choi 		.  	 	 		\textit{Ann Oper Res}  		2018. 265 p. .  	 
\bibitem[Tao et al. (2019)]{b23}\label{b23} 	 		‘LHF: A new archive based approach to accelerate massive small \textunderscore les access performance in HDFS’.  		 			W Tao 		,  		 			Y Zhai 		,  		 			J Tchaye-Kondi 		.  	 	 		\textit{Proc. 5th IEEE Int. Conf. Big Data Service Appl},  		 (5th IEEE Int. Conf. Big Data Service Appl)  		Apr. 2019. p. .  	 
\bibitem[Wang et al. ()]{b12}\label{b12} 	 		‘MOSM: An approach for efficient storing massive small files on Hadoop’.  		 			K Wang 		,  		 			Y Yang 		,  		 			X Qiu 		,  		 			Z Gao 		.  	 	 		\textit{2017 IEEE 2nd International Conference on Big Data Analysis (ICBDA)},  		 (Beijing, China)  		2017. p. .  	 
\bibitem[Prasanna and Kumar (2016)]{b14}\label{b14} 	 		‘Optimization Scheme for Storing and Accessing Huge Number of Small Files on HADOOP Distributed File System’.  		 			L Prasanna 		,  		 			Kumar 		.  	 	 		\textit{International Journal on Recent and Innovation Trends in Computing and Communication}  		Feb. 2016. 4  (2)  p. .  	 
\bibitem[He et al. (2016)]{b21}\label{b21} 	 		‘Optimization strategy of Hadoop small\textunderscore le storage for big data in healthcare’.  		 			H He 		,  		 			Z Du 		,  		 			W Zhang 		,  		 			A Chen 		.  	 	 		\textit{J. Supercomput}  		Aug. 2016. 72  (10)  p. .  	 
\bibitem[Peng et al. (2018)]{b8}\label{b8} 	 		 			Jian-Feng \& Peng 		,  		 			Wen-Guo \& Wei 		,  		 			Hui-Min \& Zhao 		,  		 			Dai 		,  		 			Gui-Yuan \& Qing-Yun \& Xie 		,  		 			Jun Cai 		,  		 			Kejing He 		.  		 \xref{http://dx.doi.org/Proceedings.10.1007/978-3-030-00563-4_50}{Proceedings.10.1007/978-3-030-00563-4\textunderscore 50}.  	 	 		\textit{Hadoop Massive Small File Merging Technology Based on Visiting Hot-Spot and Associated File Optimization: 9th International Conference},  		 (Xi'an, China)  		2018. 2018. July 7-8, 2018.  	 
\bibitem[Fu et al. (2015)]{b22}\label{b22} 	 		‘Performance optimization for managing massive numbers of small files in distributed file systems’.  		 			S Fu 		,  		 			L He 		,  		 			C Huang 		,  		 			X Liao 		,  		 			K Li 		.  	 	 		\textit{IEEE Trans. Parallel Distrib. Syst}  		Dec. 2015. 26  (12)  p. .  	 
\bibitem[Siddiqui et al. ()]{b4}\label{b4} 	 		\textit{Pseudo-Cache-Based IoT Small Files Management Framework in HDFS Cluster. Wireless Personal Communications},  		 			Isma \& Siddiqui 		,  		 			Nawab Qureshi 		,  		 			Muhammad Faseeh \& Chowdhry 		,  		 			Bhawani 		,  		 			Mohammad Uqaili 		.  		 \xref{http://dx.doi.org/113.10.1007/s11277-020-07312-3}{113.10.1007/s11277-020-07312-3}.  		2020.  	 
\bibitem[Niazi et al. ()]{b9}\label{b9} 	 		\textit{Size Matters : Improving the Performance of Small Files in Hadoop', presented at the Middleware'18},  		 			S Niazi 		,  		 			M Ronström 		,  		 			S Haridi 		,  		 			J Dowling 		.  		2018. Rennes, France: ACM. p. 14.  	 
\bibitem[Small size problem in Hadoop]{b0}\label{b0} 	 		 \url{http://blog.Cloudera.com/blog/2009/02/the-small-files-problem/}  		\textit{Small size problem in Hadoop},  		 	 
\bibitem[Solving Small size problem in Hadoop]{b1}\label{b1} 	 		\textit{Solving Small size problem in Hadoop},  		 \url{https://pastiaro.wordpress.com/2013/06/05/solving-the-small-files-problem-in-apache-hadoop-appending-and-merging-in-hdfs/}  		 	 
\bibitem[Liu ()]{b17}\label{b17} 	 		‘Storage-Optimization Method for Massive Small Files of Agricultural Resources Based on Hadoop’.  		 			Jun Liu 		.  		 \xref{http://dx.doi.org/23.634-640.10.20965/jaciii.2019.p0634}{23.634-640.10.20965/jaciii.2019.p0634}.  	 	 		\textit{Journal of Advanced Computational Intelligence and Intelligent Informatics}  		2019.  	 
\bibitem[Mu et al. (2015)]{b19}\label{b19} 	 		‘The optimization scheme research of small files storage based on HDFS’.  		 			Q Mu 		,  		 			Y Jia 		,  		 			B Luo 		.  	 	 		\textit{Proc. 8th Int. Symp. Comput. Intell. Design},  		 (8th Int. Symp. Comput. Intell. Design)  		Dec. 2015. p. .  	 
\end{bibitemlist}
 			 		 	 
\end{document}
