\documentclass[11pt,twoside]{article}\makeatletter

\IfFileExists{xcolor.sty}%
  {\RequirePackage{xcolor}}%
  {\RequirePackage{color}}
\usepackage{colortbl}
\usepackage{wrapfig}
\usepackage{ifxetex}
\ifxetex
  \usepackage{fontspec}
  \usepackage{xunicode}
  \catcode`⃥=\active \def⃥{\textbackslash}
  \catcode`❴=\active \def❴{\{}
  \catcode`❵=\active \def❵{\}}
  \def\textJapanese{\fontspec{Noto Sans CJK JP}}
  \def\textChinese{\fontspec{Noto Sans CJK SC}}
  \def\textKorean{\fontspec{Noto Sans CJK KR}}
  \setmonofont{DejaVu Sans Mono}
  
\else
  \IfFileExists{utf8x.def}%
   {\usepackage[utf8x]{inputenc}
      \PrerenderUnicode{–}
    }%
   {\usepackage[utf8]{inputenc}}
  \usepackage[english]{babel}
  \usepackage[T1]{fontenc}
  \usepackage{float}
  \usepackage[]{ucs}
  \uc@dclc{8421}{default}{\textbackslash }
  \uc@dclc{10100}{default}{\{}
  \uc@dclc{10101}{default}{\}}
  \uc@dclc{8491}{default}{\AA{}}
  \uc@dclc{8239}{default}{\,}
  \uc@dclc{20154}{default}{ }
  \uc@dclc{10148}{default}{>}
  \def\textschwa{\rotatebox{-90}{e}}
  \def\textJapanese{}
  \def\textChinese{}
  \IfFileExists{tipa.sty}{\usepackage{tipa}}{}
\fi
\def\exampleFont{\ttfamily\small}
\DeclareTextSymbol{\textpi}{OML}{25}
\usepackage{relsize}
\RequirePackage{array}
\def\@testpach{\@chclass
 \ifnum \@lastchclass=6 \@ne \@chnum \@ne \else
  \ifnum \@lastchclass=7 5 \else
   \ifnum \@lastchclass=8 \tw@ \else
    \ifnum \@lastchclass=9 \thr@@
   \else \z@
   \ifnum \@lastchclass = 10 \else
   \edef\@nextchar{\expandafter\string\@nextchar}%
   \@chnum
   \if \@nextchar c\z@ \else
    \if \@nextchar l\@ne \else
     \if \@nextchar r\tw@ \else
   \z@ \@chclass
   \if\@nextchar |\@ne \else
    \if \@nextchar !6 \else
     \if \@nextchar @7 \else
      \if \@nextchar (8 \else
       \if \@nextchar )9 \else
  10
  \@chnum
  \if \@nextchar m\thr@@\else
   \if \@nextchar p4 \else
    \if \@nextchar b5 \else
   \z@ \@chclass \z@ \@preamerr \z@ \fi \fi \fi \fi
   \fi \fi  \fi  \fi  \fi  \fi  \fi \fi \fi \fi \fi \fi}
\gdef\arraybackslash{\let\\=\@arraycr}
\def\@textsubscript#1{{\m@th\ensuremath{_{\mbox{\fontsize\sf@size\z@#1}}}}}
\def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}}
\def\abbr{}
\def\corr{}
\def\expan{}
\def\gap{}
\def\orig{}
\def\reg{}
\def\ref{}
\def\sic{}
\def\persName{}\def\name{}
\def\placeName{}
\def\orgName{}
\def\textcal#1{{\fontspec{Lucida Calligraphy}#1}}
\def\textgothic#1{{\fontspec{Lucida Blackletter}#1}}
\def\textlarge#1{{\large #1}}
\def\textoverbar#1{\ensuremath{\overline{#1}}}
\def\textquoted#1{‘#1’}
\def\textsmall#1{{\small #1}}
\def\textsubscript#1{\@textsubscript{\selectfont#1}}
\def\textxi{\ensuremath{\xi}}
\def\titlem{\itshape}
\newenvironment{biblfree}{}{\ifvmode\par\fi }
\newenvironment{bibl}{}{}
\newenvironment{byline}{\vskip6pt\itshape\fontsize{16pt}{18pt}\selectfont}{\par }
\newenvironment{citbibl}{}{\ifvmode\par\fi }
\newenvironment{docAuthor}{\ifvmode\vskip4pt\fontsize{16pt}{18pt}\selectfont\fi\itshape}{\ifvmode\par\fi }
\newenvironment{docDate}{}{\ifvmode\par\fi }
\newenvironment{docImprint}{\vskip 6pt}{\ifvmode\par\fi }
\newenvironment{docTitle}{\vskip6pt\bfseries\fontsize{22pt}{25pt}\selectfont}{\par }
\newenvironment{msHead}{\vskip 6pt}{\par}
\newenvironment{msItem}{\vskip 6pt}{\par}
\newenvironment{rubric}{}{}
\newenvironment{titlePart}{}{\par }

\newcolumntype{L}[1]{){\raggedright\arraybackslash}p{#1}}
\newcolumntype{C}[1]{){\centering\arraybackslash}p{#1}}
\newcolumntype{R}[1]{){\raggedleft\arraybackslash}p{#1}}
\newcolumntype{P}[1]{){\arraybackslash}p{#1}}
\newcolumntype{B}[1]{){\arraybackslash}b{#1}}
\newcolumntype{M}[1]{){\arraybackslash}m{#1}}
\definecolor{label}{gray}{0.75}
\def\unusedattribute#1{\sout{\textcolor{label}{#1}}}
\DeclareRobustCommand*{\xref}{\hyper@normalise\xref@}
\def\xref@#1#2{\hyper@linkurl{#2}{#1}}
\begingroup
\catcode`\_=\active
\gdef_#1{\ensuremath{\sb{\mathrm{#1}}}}
\endgroup
\mathcode`\_=\string"8000
\catcode`\_=12\relax

\usepackage[a4paper,twoside,lmargin=1in,rmargin=1in,tmargin=1in,bmargin=1in,marginparwidth=0.75in]{geometry}
\usepackage{framed}

\definecolor{shadecolor}{gray}{0.95}
\usepackage{longtable}
\usepackage[normalem]{ulem}
\usepackage{fancyvrb}
\usepackage{fancyhdr}
\usepackage{graphicx}
\usepackage{marginnote}

\renewcommand{\@cite}[1]{#1}


\renewcommand*{\marginfont}{\itshape\footnotesize}

\def\Gin@extensions{.pdf,.png,.jpg,.mps,.tif}

  \pagestyle{fancy}

\usepackage[pdftitle={Using Unlabeled Data for Increasing Low-Shot Classification Accuracy of Relevant and Open-Set Irrelevant Images},
 pdfauthor={}]{hyperref}
\hyperbaseurl{}

	 \paperwidth210mm
	 \paperheight297mm
              
\def\@pnumwidth{1.55em}
\def\@tocrmarg {2.55em}
\def\@dotsep{4.5}
\setcounter{tocdepth}{3}
\clubpenalty=8000
\emergencystretch 3em
\hbadness=4000
\hyphenpenalty=400
\pretolerance=750
\tolerance=2000
\vbadness=4000
\widowpenalty=10000

\renewcommand\section{\@startsection {section}{1}{\z@}%
     {-1.75ex \@plus -0.5ex \@minus -.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\Large\bfseries}}
\renewcommand\subsection{\@startsection{subsection}{2}{\z@}%
     {-1.75ex\@plus -0.5ex \@minus- .2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\Large}}
\renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}%
     {-1.5ex\@plus -0.35ex \@minus -.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\large}}
\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}%
     {-1ex \@plus-0.35ex \@minus -0.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\normalsize}}
\renewcommand\subparagraph{\@startsection{subparagraph}{5}{\parindent}%
     {1.5ex \@plus1ex \@minus .2ex}%
     {-1em}%
     {\reset@font\normalsize\bfseries}}


\def\l@section#1#2{\addpenalty{\@secpenalty} \addvspace{1.0em plus 1pt}
 \@tempdima 1.5em \begingroup
 \parindent \z@ \rightskip \@pnumwidth 
 \parfillskip -\@pnumwidth 
 \bfseries \leavevmode #1\hfil \hbox to\@pnumwidth{\hss #2}\par
 \endgroup}
\def\l@subsection{\@dottedtocline{2}{1.5em}{2.3em}}
\def\l@subsubsection{\@dottedtocline{3}{3.8em}{3.2em}}
\def\l@paragraph{\@dottedtocline{4}{7.0em}{4.1em}}
\def\l@subparagraph{\@dottedtocline{5}{10em}{5em}}
\@ifundefined{c@section}{\newcounter{section}}{}
\@ifundefined{c@chapter}{\newcounter{chapter}}{}
\newif\if@mainmatter 
\@mainmattertrue
\def\chaptername{Chapter}
\def\frontmatter{%
  \pagenumbering{roman}
  \def\thechapter{\@roman\c@chapter}
  \def\theHchapter{\roman{chapter}}
  \def\thesection{\@roman\c@section}
  \def\theHsection{\roman{section}}
  \def\@chapapp{}%
}
\def\mainmatter{%
  \cleardoublepage
  \def\thechapter{\@arabic\c@chapter}
  \setcounter{chapter}{0}
  \setcounter{section}{0}
  \pagenumbering{arabic}
  \setcounter{secnumdepth}{6}
  \def\@chapapp{\chaptername}%
  \def\theHchapter{\arabic{chapter}}
  \def\thesection{\@arabic\c@section}
  \def\theHsection{\arabic{section}}
}
\def\backmatter{%
  \cleardoublepage
  \setcounter{chapter}{0}
  \setcounter{section}{0}
  \setcounter{secnumdepth}{2}
  \def\@chapapp{\appendixname}%
  \def\thechapter{\@Alph\c@chapter}
  \def\theHchapter{\Alph{chapter}}
  \appendix
}
\newenvironment{bibitemlist}[1]{%
   \list{\@biblabel{\@arabic\c@enumiv}}%
       {\settowidth\labelwidth{\@biblabel{#1}}%
        \leftmargin\labelwidth
        \advance\leftmargin\labelsep
        \@openbib@code
        \usecounter{enumiv}%
        \let\p@enumiv\@empty
        \renewcommand\theenumiv{\@arabic\c@enumiv}%
	}%
  \sloppy
  \clubpenalty4000
  \@clubpenalty \clubpenalty
  \widowpenalty4000%
  \sfcode`\.\@m}%
  {\def\@noitemerr
    {\@latex@warning{Empty `bibitemlist' environment}}%
    \endlist}

\def\tableofcontents{\section*{\contentsname}\@starttoc{toc}}
\parskip0pt
\parindent1em
\def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}}
\newenvironment{reflist}{%
  \begin{raggedright}\begin{list}{}
  {%
   \setlength{\topsep}{0pt}%
   \setlength{\rightmargin}{0.25in}%
   \setlength{\itemsep}{0pt}%
   \setlength{\itemindent}{0pt}%
   \setlength{\parskip}{0pt}%
   \setlength{\parsep}{2pt}%
   \def\makelabel##1{\itshape ##1}}%
  }
  {\end{list}\end{raggedright}}
\newenvironment{sansreflist}{%
  \begin{raggedright}\begin{list}{}
  {%
   \setlength{\topsep}{0pt}%
   \setlength{\rightmargin}{0.25in}%
   \setlength{\itemindent}{0pt}%
   \setlength{\parskip}{0pt}%
   \setlength{\itemsep}{0pt}%
   \setlength{\parsep}{2pt}%
   \def\makelabel##1{\upshape ##1}}%
  }
  {\end{list}\end{raggedright}}
\newenvironment{specHead}[2]%
 {\vspace{20pt}\hrule\vspace{10pt}%
  \phantomsection\label{#1}\markright{#2}%

  \pdfbookmark[2]{#2}{#1}%
  \hspace{-0.75in}{\bfseries\fontsize{16pt}{18pt}\selectfont#2}%
  }{}
      \def\TheFullDate{1970-01-01 (revised: 01 January 1970)}
\def\TheID{\makeatother }
\def\TheDate{1970-01-01}
\title{Using Unlabeled Data for Increasing Low-Shot Classification Accuracy of Relevant and Open-Set Irrelevant Images}
\author{}\makeatletter 
\makeatletter
\newcommand*{\cleartoleftpage}{%
  \clearpage
    \if@twoside
    \ifodd\c@page
      \hbox{}\newpage
      \if@twocolumn
        \hbox{}\newpage
      \fi
    \fi
  \fi
}
\makeatother
\makeatletter
\thispagestyle{empty}
\markright{\@title}\markboth{\@title}{\@author}
\renewcommand\small{\@setfontsize\small{9pt}{11pt}\abovedisplayskip 8.5\p@ plus3\p@ minus4\p@
\belowdisplayskip \abovedisplayskip
\abovedisplayshortskip \z@ plus2\p@
\belowdisplayshortskip 4\p@ plus2\p@ minus2\p@
\def\@listi{\leftmargin\leftmargini
               \topsep 2\p@ plus1\p@ minus1\p@
               \parsep 2\p@ plus\p@ minus\p@
               \itemsep 1pt}
}
\makeatother
\fvset{frame=single,numberblanklines=false,xleftmargin=5mm,xrightmargin=5mm}
\fancyhf{} 
\setlength{\headheight}{14pt}
\fancyhead[LE]{\bfseries\leftmark} 
\fancyhead[RO]{\bfseries\rightmark} 
\fancyfoot[RO]{}
\fancyfoot[CO]{\thepage}
\fancyfoot[LO]{\TheID}
\fancyfoot[LE]{}
\fancyfoot[CE]{\thepage}
\fancyfoot[RE]{\TheID}
\hypersetup{citebordercolor=0.75 0.75 0.75,linkbordercolor=0.75 0.75 0.75,urlbordercolor=0.75 0.75 0.75,bookmarksnumbered=true}
\fancypagestyle{plain}{\fancyhead{}\renewcommand{\headrulewidth}{0pt}}

\date{}
\usepackage{authblk}

\providecommand{\keywords}[1]
{
\footnotesize
  \textbf{\textit{Index terms---}} #1
}

\usepackage{graphicx,xcolor}
\definecolor{GJBlue}{HTML}{273B81}
\definecolor{GJLightBlue}{HTML}{0A9DD9}
\definecolor{GJMediumGrey}{HTML}{6D6E70}
\definecolor{GJLightGrey}{HTML}{929497} 

\renewenvironment{abstract}{%
   \setlength{\parindent}{0pt}\raggedright
   \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}}
   \vskip16pt
   \textcolor{GJBlue}{\large\bfseries\abstractname\space}
}{%   
   \vskip8pt
   \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}}
   \vskip16pt
}

\usepackage[absolute,overlay]{textpos}

\makeatother 
      \usepackage{lineno}
      \linenumbers
      
\begin{document}

             \author[1]{Spiridon  Kasapis}

             \author[2]{Geng  Zang}

             \author[3]{Jonathon M.  Smereka}

             \author[4]{Nickolas  Vlahopoulos}

             \affil[1]{  University of Michigan}

\renewcommand\Authands{ and }

\date{\small \em Received: 1 January 1970 Accepted: 1 January 1970 Published: 1 January 1970}

\maketitle


\begin{abstract}
        


In search, exploration, and reconnaissance tasks performed with autonomous ground vehicles, an image classification capability is needed for specifically identifying targeted objects (relevant classes) and at the same time recognize when a candidate image does not belong to anyone of the relevant classes (irrelevant images). In this paper, we present an open-set low-shot classifier that uses, during its training, a modest number (less than 40) of labeled images for each relevant class, and unlabeled irrelevant images that are randomly selected at each epoch of the training process. The new classifier is capable of identifying images from the relevant classes, determining when a candidate image is irrelevant, and it can further recognize categories of irrelevant images that were not included in the training (unseen). The proposed low-shot classifier can be attached as a top layer to any pre-trained feature extractor when constructing a Convolutional Neural Network.

\end{abstract}


\keywords{semi-supervised learning, open-set classification, neural networks, receiver operating characteristic.}

\begin{textblock*}{18cm}(1cm,1cm) % {block width} (coords) 
\textcolor{GJBlue}{\LARGE Global Journals \LaTeX\ JournalKaleidoscope\texttrademark}
\end{textblock*}

\begin{textblock*}{18cm}(1.4cm,1.5cm) % {block width} (coords) 
\textcolor{GJBlue}{\footnotesize \\ Artificial Intelligence formulated this projection for compatibility purposes from the original article published at Global Journals. However, this technology is currently in beta. \emph{Therefore, kindly ignore odd layouts, missed formulae, text, tables, or figures.}}
\end{textblock*}


\let\tabcellsep& 	 	 		 
\section[{I. Introduction}]{I. Introduction}\par
xtensive research in the field of machine learning has been progressively improving the performance of object recognition algorithms which achieve impressive results on a variety of multiclass classification tasks \hyperref[b14]{[15,}\hyperref[b16]{17,}\hyperref[b23]{24]}. Especially in search, exploration, and reconnaissance applications where object recognition methods have been concentrated on a closed-set setting where all testing samples belong to one of the classes that the classifier has been trained on \hyperref[b28]{[29]}. The limited finite number of classes which are the target of inspection need to be detected out of the infinite object classes that are encountered in unconstrained environment.\par
To tackle this challenge, efforts have been made to endow Convolutional Neural Networks (CNNs) the innate human brain capability to identify objects they are trained on while deliberately discarding objects of no interest. Lately, the introduction of open-set classification \hyperref[b19]{[20,}\hyperref[b31]{31,}\hyperref[b30]{30]} has introduced an ability to correctly identify images as unknown test objects that do not belong to any known classes, as opposed to falsely classifying them in one of the known classes (i.e., classes that the model has been trained on). More specifically, \hyperref[b27]{[28,}\hyperref[b9]{10]} defines open-set classification as the problem of balancing the known space  {\ref (specialization)} and unknown open space (generalization) of the model. Examples such as out-of distribution detection \hyperref[b17]{[18]} and realistic classification \hyperref[b25]{[26]} show the interest in the concept of open-set recognition \hyperref[b3]{[4]} while showing that CNNs can be trained to reject examples that have not been seen during training or are too hard to classify.\par
Recently, works on video object discovery \hyperref[b33]{[33]} go against the closed-set assumption that each image during inference belongs to one of the fixed number of relevant classes. In \hyperref[b33]{[33]} the terminology of relevant and irrelevant is introduced and is used in this paper since it aligns with the definitions stated in the Abstract. In most real-life applications this closedset assumption is uncommon and ideal, therefore recently proposed methods \hyperref[b3]{[4]} are subject to an open-set condition where images not seen during training should be classified into irrelevant or unseen classes. Consequentially, in this work we introduce the splitting of testing samples in three categories: (a) relevant; labeled samples used during train-Figure  {\ref 1}: Schematic of the two parts of our network. We feed to the Pretrained Network labeled "Relevant" images and unlabeled "Irrelevant" images. For each image our proposed classifier produces class score vectors that get classified using a threshold criterion and Receiver Operating Characteristic (ROC), with accuracy much greater than already existing techniques, especially for the irrelevant dataset.\par
ing, (b) irrelevant; unlabeled samples used during training and (c) irrelevant but also unseen; for categories of images that are not seen during training and should be identified as irrelevant.\par
Another challenge the visual recognition community faces is the absence of labeled examples. Especially in military applications having large labeled datasets is an unreal expectation as needs and mission tools used for search and reconnaissance evolve. An open-set recognizer will face limitations such as the absence of large amounts of training samples, thus an open-set recognition technique that simultaneously supports the few-shot setting is needed. Therefore, in this paper we propose a low-shot solution to the problem of open-set recognition which considers exclusively the classification layer of a CNN.\par
Specifically, we present an approach on significantly improving the performance of a simple, time efficient, one-layer classifier on recognising labeled (relevant) images along with non-labeled (irrelevant and unseen as mentioned above) images (Figure  {\ref 1}). The ability to specifically recognize a number (of the order of 50) of relevant classes and also identify when an image does not belong to any of them in a labelinexpensive way is one of the main motivations for low-shot open-set recognition.\par
Efforts with similar goals have been concentrated on the training of the entire CNN. For example, the PEELER algorithm \hyperref[b24]{[25]} combines the random selection of a set of novel classes per episode, a loss that maximizes the posterior entropy for examples of those classes, and a new metric learning formulation in order to train the weights of a CNN in such a manner that it can recognize images of a limited amount of classes \hyperref[b19]{( 20)} that are unseen during training. Dhamija et al. \hyperref[b6]{[7]} proposes the introduction of two loss functions that are designed to maximize entropy for unknown inputs while increasing separation in deep feature space by modifying magnitudes of known and unknown samples. Although the work of Dhamija et al. introduces the concept of unknown sample recognition like we do, the number of recognizable classes is still very limited compared to the testing done in our method.\par
Both of the aforementioned algorithms train the entirety of the CNN, unlike the methods proposed by Kozerawski and Turk \hyperref[b22]{[23]} which can augment any few shot learning method without requiring retraining in order to work in a few-shot multiclass open-set setting.\par
Although not concerned with one-class classification, a similar approach is followed in our work too, where we utilize a pre-trained feature extractor (such as the ones publicly available by PyTorch 1 ) and propose an independent open-set low-shot classification method which can augment any existing feature extractor.\par
To explore the open-set low-shot problem in a holistic, non-specific and easily applicable way, we concentrate only on the training of the classification matrix (matrix used to turn the feature vector to a probability vector in Figure  {\ref 1}), using the pre-trained ResNet feature extractors \hyperref[b14]{[15]} discussed in Section 2. We reduce the image matrices to feature vectors \hyperref[b7]{[8,}\hyperref[b0]{1]} which are then used in Section 4 to train the classifier with the help of the analytic derivative of our loss function and a unique, partially labeled, target matrix. In Section 5, we use the classification variability statistics and a Receiver Operating Characteristic (ROC) curve as a method to calculate threshold scores for each relevant class. An approach that uses random selections of unlabeled irrelevant images during each epoch of the classifier training is introduced. Testing datasets are used in Section 3 for determining the ability to effectively classify all Relevant, Irrelevant and Unseen datasets. In the last Section we make some closing remarks on our work presented in this paper.\par
In summary, the contributions of this work are the following:? We present a novel open-set low-shot (OSLS)\par
classification method which can be added as the top layer to any pre-trained feature extractor in order to create a CNN that can classify images in relevant classes and also determine if an image does not belong to any of the relevant classes.  
\section[{II. Feature Extractor}]{II. Feature Extractor}\par
Deep Residual Networks have been proven to be a very effective in mapping images to a meaningful feature space, especially when trained from large datasets \hyperref[b32]{[32]}. In this work we use ResNet18 and ResNet34 \hyperref[b15]{[16]} to map the sample images to the vector space. Both architectures produce 512-long feature vectors which compared to deeper network feature products lead to a shorter algorithm running time. The different types of ResNets we used, although not very different, will be discussed in Section 3. The weights were trained using the ImageNet1k dataset \hyperref[b5]{[6]} which involves a large-scale ontology of images. The development of the feature extractor itself is out of the scope of this paper and pre trained feature extractors available by PyTorch are used.\par
Before providing the training images feature vectors to the OSLS classifier we normalize them using the following equation: \hyperref[b0]{(1)} Where is a 512-long vector feature map and are its respective maximum and minimum values in vector form. This type of basic normalization constrains the values from 0 to 1. We apply the normalization to prevent the Exponential Loss and its Derivative in Equation  {\ref 9}and Equation  {\ref 11}, respectively, from gaining extremely high values. Additionally, we demonstrate in Section 6 that this type of normalization significantly increases our method's classification accuracy compared to the more popular softmax normalization. It can be argued that the use of softmax normalization yields poor solutions for open-set recognition as it tends to overfit on the training classes. 
\section[{III. Datasets}]{III. Datasets}\par
The proposed OSLS classification method is used on a variety of training and testing examples, each using different sample arrangements. To explore the capabilities of the proposed method in different settings, two different datasets are being used: the Caltech256 \hyperref[b10]{[11]} and a custom Mixed dataset.\par
To explain with more clarity our method and results, we describe the way the Caltech256 dataset is split in two groups. Caltech256 is an open source dataset, it consists of 256 different image classes and has been recently used a lot as a benchmark for a variety of machine learning applications \hyperref[b1]{[2,}\hyperref[b8]{9]}.\par
Similar to our selection of ResNets, we use an open source and broadly used dataset in order to make our example and results as general and less task specific as possible. As we intend to produce work that is going to be used in the future for specific applications, to give a hint on how the method can be geared towards recognition in unique environments the eight infrared classes (Figure \hyperref[fig_1]{3}) which are available from the Military Sensing Information Analysis Center (SENSIAC) Automatic Target Recognition (ATR) database \hyperref[b34]{[34]} are used in a number of our tests.  Finally, in order to explore the behavior of our method on unique and very different environments from the ones present in the Caltech256 dataset, we created our own infrared (IR) combat vehicle image group by taking snapshots from the publicly available IR videos provided in the Military Sensing Information Analysis Center (SENSIAC) ATR database (examples displayed in Figure \hyperref[fig_1]{3}). The new data product is composed of the same amount of pictures with the one in Figure \hyperref[fig_0]{2}, with the exception that the first 8 relevant image classes are infrared instead of Caltech 256 pictures. Although only The second dataset we train and test our classification method on is the Mixed dataset. The relevant group is composed of 50 select classes from the ImageNet \hyperref[b5]{[6]} dataset and includes pictures of vehicles, aircraft, humans and weapons. This group will serve as the target images that are expected to be recognised. On the other hand, the irrelevant group is composed of 50 classes from the MIT Places \hyperref[b35]{[35]} dataset which includes a variety of outdoor scenery pictures such as buildings and natural environment. The choice of these two datasets is deliberate as in our application we are trying to recognize objects in a scene and push away scenes that have no relevant objects. Each class on the relevant part of the mixed dataset is comprised of 1,300 images, while each irrelevant class has available 13,000 pictures on average. The imbalance between relevant and irrelevant images is representative of the imbalance in the unlabeled data captured in the field which will contain many more irrelevant objects compared to targeted classes. From every class in both groups, 10 images are reserved for testing the accuracy of the various methods which are compared after the training of the classifier has been completed. When using the Mixed dataset in this work, a part of the irrelevant pictures will be reserved and used as unseen samples (Figure  {\ref 9}), images that have not been seen during training but have to be recognized by the classifier in the same way as the irrelevant.\par
Examples of the mixed dataset images are presented on Figure \hyperref[fig_2]{4} while a complete list of the classes in alphabetical order is presented in the Appendix. The way the mixed dataset is utilised for training and testing the low-shot classifier is discussed in Section 5. 
\section[{IV. Low-Shot Classifier Training}]{IV. Low-Shot Classifier Training}\par
The two integral parts of our classifier training process are the target matrix and the loss function. Our training goal is to tweak the initially randomized weight matrix in such a manner that when multiplying it with a testing feature map, it produces a score matrix whose largest value is the desired class element.\par
In machine learning, a fully connected layer performs the following calculation:\par
(\par
Where W is weighting matrix of the classifier, F is the feature map matrix, V is the bias vector and f is the activation function. A Singular Value Decomposition (SVD) method solves our matrix equations \hyperref[b20]{[21,}\hyperref[b18]{19]}. The pseudo-inverse method calculation results as:\par
(3)\par
Here, is the feature maps,, is the weight matrix which we desire to train, and ?H is the target, the ideal outcome for the score matrix. Our MATLAB implementation handles the training one class at the time, therefore is a matrix and , is a long vector for each class, where is the number of training images in every epoch and = 512 is the length of the feature vectors (constant). With no use of the bias vector, and the reversed order of and , to account for the row-column switch, in SVD we calculate the , matrix one vector (class) at a time therefore essentially solving for the least square solution of:\par
(4)\par
When the exact solution does not exist, which means that is not a full-rank square matrix, we get approximate solutions as: Therefore the approximation error is: \hyperref[b5]{(6)} and in a least-square approach the loss function is: \hyperref[b6]{(7)} substituting Equation  {\ref 6}results:\par
We introduce an exponential version of the least square solution in order to explore a new, faster converging loss function based on \hyperref[b21]{[22]}. Our new squared-exponential loss function is: \hyperref[b8]{(9)} therefore the gradient can be proven analytically to be: \hyperref[b9]{(10)} and the gradient vector is: \hyperref[b10]{(11)} There are two main reasons for choosing this loss function. A squared-exponential function is easy to differentiate analytically and the differentiation is applied to the linear algebra form implemented in the MATLAB code. Note that the dot operator in Equation  {\ref 11}, i.e., . , used with multiplication in MATLAB, creates element wise operations. Compared to other differentiable functions we tested, the square-exponential was the one to converge faster and in a steady way. A problem we encountered, which we solved by normalizing the feature maps as described in Equation  {\ref 1}, is that because of the nature of the function, for numbers greater than 1, the Loss would result in extremely high values.\par
The gradient matrix in Equation 11 is then multiplied by a learning rate( ) and added to the weight matrix ( , ), repeating this sequence for every epoch. The steps taken towards training the classifier matrix are therefore all independent from machine learning libraries or functions. Although many different loss functions that get differentiated in a semi-analytic fashion are being used by machine learning libraries, we concentrated our efforts on not using any existing libraries to create a stand-alone method. Therefore the squared-exponential loss function is a good fit.\par
As in most machine learning applications, the update mechanism used towards convergence is some variation of a normal gradient descent equation. In our specific case we use: \hyperref[b11]{(12)} Here, in every epoch , , gets updated by subtracting from it the product of the learning rate [ and the gradient matrix. We obtain our learning rate using an algorithm inspired by Iterative Shrinkage-Thresholding Algorithm (ISTA) \hyperref[b2]{[3]}. We begin with calculating a pseudo-loss which is going to be compared with the actual loss to determine whether the learning rate needs to be decreased or kept as specified on the previous epoch. This iterative method progressively decreases the learning rate as we approach closer to the desired optimal point. The last, and most unique part about our classification mAs in most machine learning applications, the update mechanism used towards convergence is some variation of a normal gradient descent equation. In our specific case we use: ethod is our target. As mentioned in the Section 1, the uniqueness of our approach relies on the fact that we make use of unlabeled images during the training of the weight matrix. This is done by extending a typical onehot encoding \hyperref[b13]{[14]} matrix to also include class score distributions as targets for the Irrelevant images. Labeled images have arrays of zeros and a unit value on the correct class element as targets.\par
() = ? ? = || || = =1 2 = =1 ( ? ? ) 2 = =1 ( =1 ? ) 2 = =1 2 = =1 ( =1 ? ) 2 = =1 2 2 = ( . * 2 ) * ? = ? ? ? ? ? ? ? ? ? ? ? ? ? ? 1 0 0 0 1 0 0 0 1 ?0.2 ?0.2 ?0.2 ?0.2 ?0.2 ?0.2 ?0.2 ?0.2 ?0.2 ? ? ? ? ? ? ? ? ? ? ? ? ? ? +1 = ? 1 2\textbf{13}\par
Irrelevant pictures belong to none of the classes therefore the score for each class should be zero. By experimentation we concluded that the irrelevant target that works best should be a slight negative value, such as -0.2. This intuition matches some of the binary classification work that has been done on Support Vector Machines' (SVM) correlation filters, where 0 and 1 were not as separable as a negative value (-0.1, -1) and 1 \hyperref[b36]{[36,}\hyperref[b4]{5]}. As an example, if a training dataset was consisted of six pictures, half of them labeled and half of them unlabeled, and the labeled ones were members of three different classes, our target matrix would look as follows:\par
We train the weight matrix in such a way that during evaluation the irrelevant images class score vector values are spread equally between the classes and acquire values as close to zero as possible. This helps the Irrelevant pictures to score less than the respective class threshold.\par
Along with using this target oriented training procedure, we also increase our recognition accuracy by calculating our threshold scores using the ROC method as explained in Section 5. 
\section[{V. Low-Shot Classifier and Roc Threshold Calculation}]{V. Low-Shot Classifier and Roc Threshold Calculation}\par
The multiplication between a feature vector and a weight matrix yields a class score vector. Neural Network classification theory uses the highest score (Top 1, Top 3 or Top 5 have been used too) to group the images into classes. We extend this criterion to make it applicable when unlabeled images are present by introducing a Threshold Score ( value for each class.\par
The value serves as a binary discriminating test in order to group pictures in the Relevant and Irrelevant bins. As mentioned above, we don't only need to divide pictures in the two groups, we also want the relevant group pictures to be normally classified in their respective class.\par
It is important to note that the is calculated during the training of the OSLS. We need the to be pre-calculated before we start evaluating our testing dataset. Once the training has been completed and the is known, the classification process runs as follows: a) The testing image runs through the classifier and scores, which denote the likelihood of the image belonging to each class, are calculated. b) The image is assigned to the class with highest score. c) The score of the assigned image is compared to the of the class where it was assigned. If it is higher then it is considered as a member of the class. If lower, it is determined to be an irrelevant image.  To maximize the combined accuracy we use the ROC curve \hyperref[b26]{[27,}\hyperref[b12]{13,}\hyperref[b11]{12]} to chose our threshold values. The same Way we would do with a Normal Threshold, the ROC Threshold is going to be calculated right after training and before testing, using explicitly the training data.\par
To demonstrate the need of using the ROC we graph the ROC curves of five, unique compared to each other, classes of pictures that we trained our classifier on. All five classes were part of the labeled dataset (relevant classes). We can see in Figure \hyperref[fig_6]{7}  False Relevant is the ratio of pictures that were irrelevant but were predicted as relevant, over the total number of incorrectly classified pictures.\par
In our analysis we focus on the classification of Relevant pictures, which we assume to be the Positive statistical case, therefore we use the terms True Relevant Rate (TRR) and False Relevant Rate (FRR). TRR and FRR are no different than the True Positive Rate (or Sensitivity) and False Positive Rate (or Fall-Out) respectively, used in statistical analysis. Therefore we define:(14) = + and = +\par
In Figure \hyperref[fig_6]{7} it is obvious that the IR Human class is so unique that the classifier does not have any trouble distinguishing it from the rest of the dataset, therefore as seen in the figure above its AUC equals to 1 and the ROC does not have much effect on its cumulative accuracy. Different classes though present different levels of difficulty for our classifier. The Chimp class as seen above, has an AUC of 0.91, which means that the ROC can significantly improve its cumulative accuracy if a is picked wisely.\par
The Normal Threshold would pick the point on the graph where the False Relevant Rate is minimum for a True Relevant Rate of 1, hence, for the Chimp graph, the ¹0.57. 1.00º point. Using our ROC algorithm we can pick any other point on the graph, such as the ¹0.20. 0.84º point which is the one further away from the blue line that represents a random guess. By doing this, although we slightly decreased our TRR, we get a great increase in FRR, which results to a significantly higher cumulative accuracy. This accuracy increase is going to be clearly presented and discussed in the next Section.\par
Tables 1, 2 and 3 demonstrate the usefulness of the ROC method in classifying Relevant images and rejecting an image if it is Irrelevant. In order to highlight the ROC capabilities, we compare our results to the two baseline methods, the "No Irrelevant" and the "+1 Class".\par
The first baseline result (No Irrelevant), was produced by training the classifier only on labeled images of the relevant classes. This is the case where although we have unlabeled images for the irrelevant classes, we do not use them, expecting the labeled images to have enough meaningful features to accommodate recognizing the irrelevant ones. To evaluate this method, we use our Normal Threshold Score Criterion discussed above where we set the lowest correct relevant training score as the threshold for each class. During testing, if the image's highest class score is larger than the respective threshold then its classified as Relevant, if not as irrelevant.\par
The second baseline result, which we call "+1 Class", was generated by training the classifier to recognize the relevant classes along with one extra class which encapsulates all irrelevant images. During the training of the classifier, all unlabeled images of the Irrelevant classes were assigned to an extra class. The evaluation is being done by simply comparing the highest scoring index of every image with the correct target.\par
Table  {\ref 1} shows how the baseline methods scored for both relevant and irrelevant images compared to the Low-Shot Classifier, with and without applying the ROC optimization for the Top-1 selections. 
\section[{Table 1: Low-Shot Classifier Compared to Baseline}]{Table 1: Low-Shot Classifier Compared to Baseline}\par
Examples for the Top-1 Selections\par
As "Normal" we describe the dataset consisted of 50 relevant and 50 irrelevant Caltech256 classes and as "Infrared" the dataset where we have substituted 8 of the relevant classes with IR ones. Both datasets are described in Section 3. The Low-Shot Classifier results are obtained by running the algorithm described up until Section 6 and the Low-Shot Classifier with ROC by adding the ROC extension. "R" and "I" are the relevant and irrelevant classification accuracy re-spectively. The numbers shown in the tables are the Top-1 percentages of images that got classified correctly during evaluation. It can be observed that the baseline methods are unable to classify decently both groups of images. The +1 Class method seems to over-train the classifier on recognising the unlabeled images failing to put the labeled ones in the correct classes. This happens most likely due to the unbalanced training data, as the 51st class has as many images as the rest 50 together. On the other hand, by using only labeled images, we train the classifier to specifically recognise the labeled group, failing to filter out the unlabeled images "noise".\par
In the first row of Table  {\ref 1}, the results of our classifier without the ROC extension show that our loss function combined with our unique target matrix and the threshold score criterion can recognize equally well both labeled and unlabeled images. It is notable that for the relevant group our method loses a small amount of accuracy compared to the label-specific baseline method but does substantially better in identifying irrelevant images. The ROC method greatly increases the unlabeled images recognition, to the modest expense of the labeled images. The table shows the importance of using the ROC to greatly increase the cumulative accuracy. Our ROC classifier increases by 12\% the cumulative recognition scores compared to the +1 Class method and by 25.4\% compared to the label exclusive transfer learning method.\par
The described results are also depicted in the Accuracy Comparison Graph in Figure \hyperref[fig_7]{8}. For every method discussed we use three different ResNet10 feature extractors (BatchSGM, SGM, L2) in order to show the consistency of our classifier results.\par
With a few exceptions, no matter the feature extractor or the nature of our dataset (including infrared, including more unlabeled images), our proposed method (green data points) not only provides a higher cumulative accuracy but also eliminates the bias between labeled and unlabeled images by classifying both equally well when compared to the baseline approaches. In the graph we introduced the results of our extended datasets which consists of more unlabeled images.\par
Table  {\ref 2} offers a closer look to the comparison of the two extended datasets. 
\section[{Table 2: Extended Datasets Comparisons}]{Table 2: Extended Datasets Comparisons}\par
We follow the same notation in Table  {\ref 2} as used in Table  {\ref 1}, with the only difference being that the "+50" and "+150" Irrelevant datasets are the two expanded datasets noted in Section 3. Although it is clear in both in Table  {\ref 2} and the plot in Figure \hyperref[fig_7]{8}, that our method still scores better in a cumulative perspective, we can also observe that biases against the relevant (in Low-Shot Classifier with ROC algorithm) or irrelevant (in Low-Shot Classifier algorithm) group begin to occur when increasing the amount of irrelevant images.\par
We see that for the Low-Shot Classifier the more we increase the irrelevant to relevant ratio the worse we score on the irrelevant part. This might seem counter intuitive as we would expect that the more unlabeled images we see during training, the better we would be able to recognise them. In reality, we introduce many more feature elements on the irrelevant part, which leads to consequently eliminating their uniqueness.\par
When introducing the proposed ROC approach on the second row (Table  {\ref 2}) we do not observe the introduction of bias because the ROC threshold has been adjusted in such a way that it is non discriminating against any group (OptimalROC). On Table  {\ref 3} we present the results of our adjusted ROC Classifier being used on the +150 Irrelevant dataset. The same behavior is observed when we test the rest datasets.\par
The "Optimal ROC" and "No Irrelevant" rows correspond to Table  {\ref 2} second and fourth data rows. Putting a constraint on how much we are willing to shift the )( to limit the loss in relevant, affects negatively the irrelevant. We desire to find a percentage which during testing gives us a decent cumulative accuracy without big losses on the Relevant part. This could be imagined as turning a knob to tune our ROC Table  {\ref 3}: ROC Adjustment for the +150 Irrelevant dataset implementation. This can be specific in every application, therefore an open ended approach is adopted. A100\% constraint would be the Low-Shot Classifier without ROC, as we set our Threshold Scores to be the lowest correctly classified irrelevant picture in every class. On the table presented, a 90\% Constraint means that we ask our ROC algorithm to keep our thresholds to a value that will not hurt our correct relevant guesses more than 10\% during the calculation of the . Therefore these constraints are applied when using the training images and they differ from the percentages encountered in the testing (Table  {\ref 3}). As we can see, for the specific case we can compromise with an 18.7\% total increase instead of the the 20.5\% of the optimal case, in order to get a more equal recognition accuracy. 
\section[{VI. OSLS Classifier Results}]{VI. OSLS Classifier Results}\par
The Low-Shot Classifier is able to recognise images from the relevant classes and also identify irrelevant images from the classes it has seen during training. Ideally, during operation we desire to recognise objects that are not seen at all during training, which is the main objective of openset recognition. To achieve this we extend the capabilities of the Low-Shot Classifier described in Section 3 to recognizing unseen images resulting the OSLS Classifier. The unseen samples are the sub-group of the irrelevant classes that do not get involved in training but it is still expected that the OSLS Classifier recognises them as irrelevant. This ). By introducing this imbalance and by not repeating the same irrelevant samples in each epoch, our classifier is able to generalize better on the irrelevant part, yielding better classification accuracy for the irrelevant and unseen testing samples. In all results presented in this paper the testing images are always different than the images used during training.    All the results presented in the box plots of this text are for an OSLS classifier that is trained on 40 relevant and 40 irrelevant classes, both of which have the amount of relevant images per class specified in the x-axis. Similar works in the open-set literature \hyperref[b9]{[10,}\hyperref[b6]{7,}\hyperref[b24]{25]} are using a lower number of classes during training and testing (10 to 95 classes compared to the total number of classes used in this paper ranging between 90 and 250). To show how the OSLS Classifier performs in tests where the same order of classes are used, we vary the number of relevant and irrelevant classes used during training. Although it is of interest to recognize samples of as many classes as possible (a maximum of 40 as presented in Figure \hyperref[fig_10]{10}), by observing Table  {\ref 4} it is evident that the OSLS Classifier achieves very high Top-1 accuracy scores in situations where the relevant and irrelevant classes we are trying to detect are limited. We take as an example the case (in bold) where we train the of the proposed method we attach the classifier to deeper feature extractors. Throughout the paper the feature extractor used to test any classifier was a pretrained ResNet18 provided by PyTorch1. In Figure 11 results for a classifier similar to the one in Figure \hyperref[fig_10]{10} are presented with the only difference being that the feature vectors are produced using the deeper ResNet34. Improvements in accuracy ranging from \textunderscore  0.11 to \textunderscore  0.02 (for the 5 and 40 Images per Class cases respectively), compared to those of Figure \hyperref[fig_10]{10b}, can be observed for the relevant testing samples while virtually no improvement is observed for the irrelevant and the unseen samples. Similar results are expected if the OSLS Classifier is used as a head for deeper networks which produce feature vectors of higher quality. The improvements can be attributed to the fact that a deeper network has the ability to produce better quality feature representations.\par
The image feature representations used in this study are obtained raw, before any normalization is applied to them. As mentioned in Section 2, we use Equation 1 to normalized the input feature vectors. Figure \hyperref[fig_12]{12} exhibits a decrease in accuracy if the features are normalized using the popular Softmax normalization commonly used in classification layers. More specifically, the OSLS results in Figure \hyperref[fig_10]{10b} show an improvement compared to Figure \hyperref[fig_12]{12} that ranges from 0.19 to 0.15 for the relevant, 0.17 to 0.08 for the irrelevant and 0.26 to 0.08 for the unseen testing samples (for the 5 and 40 Images per Class cases respectively).\par
Finally, in order to demonstrate the value of the OSLS classifier, we compare it to the two baseline examples mentioned in Table  {\ref 1}. The first alternative method (Figure 13a) for classifying relevant samples along with rejecting irrelevant and unseen images is to group all the later in one class during training by assigning the extra class label to them ("+1 Class"). The second method (Figure 13b) the OSLS Classifier is compared to is a normal classification layer which is trained only on relevant images but is expected to recognize irrelevant and unseen images too ("No Irrelevant").\par
By comparing Figure \hyperref[fig_10]{10b} to Figure 13a, for a low number of samples per class, the "+1 Class" method performs equally well or in cases even better in all three categories compared to OSLS, with relevant accuracy scores ranging from 0.6 to 0.7 for the 5, 10 and 20 Images per Class cases while unseen and irrelevant recognition reaching accuracies 0.96. When enough data samples per class are available though, the OSLS method improves the relevant accuracy by 0.05 and 0.1 for the 30 and 40 images per class cases respectively. The improvement in relevant image classification that the OSLS classification (Figure \hyperref[fig_10]{10b}) achieves is significant compared to the minor ( 0.01) decrease in relevant and unseen accuracy scores.\par
The OSLS classifier is meant to be used as the final layer of any CNN that is expected to recognise samples that belong to the training classes while identifying as irrelevant images that are not relevant regardless if they originate from seen or unseen during training datasets. In order to demonstrate the versatility   (5 and 10), a normal classification layer tends to over-fit on the later. Due to this over-fitting, the ROC Threshold rejects most of the samples during testing resulting to very high ( 0.9) irrelevant and unseen and very low ( 0.53) relevant accuracy scores. When there are more training images per class, a significant increase in relevant accuracy can be observed which is followed by a decrease in irrelevant and unseen accuracy. More specifically, assuming similar specifications (normalization, loss function etc.), if a single layer classifier is trained on 40 classes, each one including 40 images, the mean relevant, irrelevant and unseen accuracies during testing are 0.78, 0.72 and 0.73 respectively. If an equal number of unlabeled images are used during training, in the manner specified by the OSLS method, the mean relevant accuracy decreases by 0.02 while the irrelevant and unseen accuracy scores increase by 0.22 and 0.21 respectively. The trade-off between a very small decrease in relevant accuracy and a ten times larger increase in both irrelevant and unseen classification performance is the best demonstration of how the OSLS Classifier can be utilized in real-life applications.\par
The proposed OSLS Classifier using the ROC Threshold Score criterion not only makes the resulting model more flexible and easy to customize depending on the needs of the datasets, but also makes the method flexible for any application. This is a specifically interesting feature of our work, as we can use the classifier as an extension to any image recognition algorithm which desires to filter out irrelevant and unseen images without the expense of labeling. 
\section[{VII. Conclusion}]{VII. Conclusion}\par
In military reconnaissance applications a capability is needed where objects of interest -such as adversary targets-are reliably distinguished from objects of no relevance. Although a modest amount of labeled examples for the targets might be available to use during the training of the classifier, labels for the irrelevant objects might be scarce or even not possible to obtain.\par
To tackle this problem in this work we present an Open-Set Low-Shot Classifier which is trained using a modest number of labeled images from the relevant classes and unlabeled irrelevant images. During testing, this information is used for determining when a candidate image is either relevant, irrelevant or even unseen during training. The OSLS Classifier performs better compared to baseline classifying approaches, is able to handle the classification of many more classes compared to similar open-set approaches in the visual recognition literature and is able to demonstrate sufficient balance with high\begin{figure}[htbp]
\noindent\textbf{2}\includegraphics[]{image-2.png}
\caption{\label{fig_0}Figure 2 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{3}\includegraphics[]{image-3.png}
\caption{\label{fig_1}Figure 3 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{4}\includegraphics[]{image-4.png}
\caption{\label{fig_2}Figure 4 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{}\includegraphics[]{image-5.png}
\caption{\label{fig_3}}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{5}\includegraphics[]{image-6.png}
\caption{\label{fig_4}Figure 5 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{6}\includegraphics[]{image-7.png}
\caption{\label{fig_5}Figure 6 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{7}\includegraphics[]{image-8.png}
\caption{\label{fig_6}Figure 7 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{8}\includegraphics[]{image-9.png}
\caption{\label{fig_7}Figure 8 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{}\includegraphics[]{image-10.png}
\caption{\label{fig_8}}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{910}\includegraphics[]{image-11.png}
\caption{\label{fig_9}Figure 9 :Figure 10 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{10}\includegraphics[]{image-12.png}
\caption{\label{fig_10}Figure 10}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{4}\includegraphics[]{image-13.png}
\caption{\label{fig_11}Table 4 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{12}\includegraphics[]{image-14.png}
\caption{\label{fig_12}Figure 12 :}\end{figure}
      			\footnote{© 2022 Global Journals} 			\footnote{( ) D © 2022 Global Journals Year 2022} 		 		\backmatter   			 
\subsection[{Acknowledgements}]{Acknowledgements}\par
We acknowledge the technical and financial support of the Automotive Research Center (ARC) in the University of Michigan in accordance with the Cooperative Agreement W56HZV-19-2-0001 of the U.S. Army CCDC Ground Vehicle Systems Center (GVSC) in Warren, MI. 			  			  				\begin{bibitemlist}{1}
\bibitem[Hendrycks and Gimpel ()]{b17}\label{b17} 	 		\textit{A baseline for detecting misclassified and out-of-distribution examples in neural networks},  		 			Dan Hendrycks 		,  		 			Kevin Gimpel 		.  		 arXiv:1610.02136.  		2016.  	 	 (arXiv preprint) 
\bibitem[Donahue]{b7}\label{b7} 	 		‘A deep convolutional activation feature for generic visual recognition’.  		 			Jeff Donahue 		.  	 	 		\textit{UC Berkeley \& ICSI}  		 	 
\bibitem[Beck and Teboulle ()]{b2}\label{b2} 	 		‘A fast iterative shrinkage-thresholding algorithm for linear inverse problems’.  		 			Amir Beck 		,  		 			Marc Teboulle 		.  	 	 		\textit{SIAM journal on imaging sciences}  		2009. 2  (1)  p. .  	 
\bibitem[Kim and Vlahopoulos ()]{b21}\label{b21} 	 		‘A multilevel optimization algorithm and a ship design application’.  		 			Hongyoon Kim 		,  		 			Nickolas Vlahopoulos 		.  	 	 		\textit{12th AIAA Aviation Technology, Integration, and Operations (ATIO) Conference and 14 th AIAA/ISSMO Multidisciplinary Analysis and Optimization Conference},  				2012. p. 5555.  	 
\bibitem[David et al. ()]{b12}\label{b12} 	 		‘A simple generalization of the area under the ROC curve for multiple class classification problems’.  		 			J David 		,  		 			Robert J Hand 		,  		 			Till 		.  	 	 		\textit{Machine learning}  		2001. 45 p. .  	 
\bibitem[Pham and Polasek]{b28}\label{b28} 	 		\textit{Algorithm for military object detection using image data},  		 			Iq Pham 		,  		 			M Polasek 		.  		p. 2014.  	 
\bibitem[Sh Shabbeer Basha ()]{b1}\label{b1} 	 		‘AutoTune: Automatically Tuning Convolutional Neural Networks for Improved Transfer Learning’.  		 			Sh Shabbeer Basha 		.  	 	 		\textit{Neural Networks}  		2021. 133 p. .  	 
\bibitem[Ge and Yu ()]{b8}\label{b8} 	 		‘Borrowing treasures from the wealthy: Deep transfer learning through selective joint fine-tuning’.  		 			Weifeng Ge 		,  		 			Yizhou Yu 		.  	 	 		\textit{Proceedings of the IEEE conference on computer vision and pattern recognition},  				 (the IEEE conference on computer vision and pattern recognition)  		2017. p. .  	 
\bibitem[Griffin et al. ()]{b10}\label{b10} 	 		\textit{Caltech-256 object category dataset},  		 			Gregory Griffin 		,  		 			Alex Holub 		,  		 			Pietro Perona 		.  		2007.  	 
\bibitem[He ()]{b14}\label{b14} 	 		‘Deep residual learning for image recognition’.  		 			Kaiming He 		.  	 	 		\textit{Proceedings of the IEEE conference on computer vision and pattern recognition},  				 (the IEEE conference on computer vision and pattern recognition)  		2016. p. .  	 
\bibitem[Dj Peres and Cancelliere ()]{b26}\label{b26} 	 		‘Derivation and evaluation of landslide-triggering thresholds by a Monte Carlo approach’.  		 			Dj Peres 		,  		 			Cancelliere 		.  	 	 		\textit{Hydrology and Earth System Sciences}  		2014. 18  (12)  p. .  	 
\bibitem[Harris and Harris ()]{b13}\label{b13} 	 		\textit{Digital design and computer architecture},  		 			David Harris 		,  		 			Sarah L Harris 		.  		2010. Morgan Kaufmann.  	 
\bibitem[Liu ()]{b24}\label{b24} 	 		‘Few-shot open-set recognition using meta-learning’.  		 			Bo Liu 		.  	 	 		\textit{Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},  				 (the IEEE/CVF Conference on Computer Vision and Pattern Recognition)  		2020. p. .  	 
\bibitem[Azizpour ()]{b0}\label{b0} 	 		‘From generic to specific deep representations for visual recognition’.  		 			Hossein Azizpour 		.  	 	 		\textit{Proceedings of the IEEE conference on computer vision and pattern recognition workshops},  				 (the IEEE conference on computer vision and pattern recognition workshops)  		2015. p. .  	 
\bibitem[Ge ()]{b9}\label{b9} 	 		\textit{Generative openmax for multiclass open set classification},  		 			Zongyuan Ge 		.  		 arXiv:1707.07418.  		2017.  	 	 (arXiv preprint) 
\bibitem[He ()]{b15}\label{b15} 	 		‘Identity mappings in deep residual networks’.  		 			Kaiming He 		.  	 	 		\textit{European conference on computer vision},  				2016. Springer. p. .  	 
\bibitem[IEEE/AIAA 33rd Digital Avionics Systems Conference (DASC) ()]{b29}\label{b29} 	 		\textit{IEEE/AIAA 33rd Digital Avionics Systems Conference (DASC)},  				2014. IEEE. p. .  	 
\bibitem[Krizhevsky et al. ()]{b23}\label{b23} 	 		‘Imagenet classification with deep convolutional neural networks’.  		 			Alex Krizhevsky 		,  		 			Ilya Sutskever 		,  		 			Geoffrey E Hinton 		.  	 	 		\textit{Advances in neural information processing systems}  		2012. 25 p. .  	 
\bibitem[Deng ()]{b5}\label{b5} 	 		‘Imagenet: A large-scale hierarchical image database’.  		 			Jia Deng 		.  	 	 		\textit{IEEE conference on computer vision and pattern recognition},  				2009. 2009. Ieee. p. .  	 
\bibitem[Jia ()]{b20}\label{b20} 	 		‘Improving training of deep neural networks via singular value bounding’.  		 			Kui Jia 		.  	 	 		\textit{Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},  				 (the IEEE Conference on Computer Vision and Pattern Recognition)  		2017. p. .  	 
\bibitem[Yu ()]{b34}\label{b34} 	 		‘Joint infrared target recognition and segmentation using a shape manifold-aware level set’.  		 			Liangjiang Yu 		.  	 	 		\textit{Sensors}  		2015. 15 p. .  	 
\bibitem[Zuo ()]{b36}\label{b36} 	 		‘Learning support correlation filters for visual tracking’.  		 			Wangmeng Zuo 		.  	 	 		\textit{IEEE transactions}  		2018. 41 p. .  	 
\bibitem[He ()]{b16}\label{b16} 	 		‘Mask r-cnn’.  		 			Kaiming He 		.  	 	 		\textit{Proceedings of the IEEE international conference on computer vision},  				 (the IEEE international conference on computer vision)  		2017. p. .  	 
\bibitem[Vishnu et al. ()]{b4}\label{b4} 	 		\textit{Maximum margin vector correlation filter},  		 			Naresh Vishnu 		,  		 			Bvk Boddeti 		,  		 			Kumar 		.  		 arXiv:1404.6031.  		2014.  	 	 (arXiv preprint) 
\bibitem[David and Hand ()]{b11}\label{b11} 	 		‘Measuring classifier performance: a coherent alternative to the area under the ROC curve’.  		 			J David 		,  		 			Hand 		.  	 	 		\textit{Machine learning}  		2009. 77 p. .  	 
\bibitem[Pernici ()]{b27}\label{b27} 	 		‘Memory based online learning of deep representations from video streams’.  		 			Federico Pernici 		.  	 	 		\textit{Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition},  				 (the IEEE Conference on Computer Vision and Pattern Recognition)  		2018. p. .  	 
\bibitem[Lalit et al. ()]{b19}\label{b19} 	 		‘Multi-class open set recognition using probability of inclusion’.  		 			P Lalit 		,  		 			Jain 		,  		 			J Walter 		,  		 			Terrance E Scheirer 		,  		 			Boult 		.  	 	 		\textit{European Conference on Computer Vision},  				2014. Springer. p. .  	 
\bibitem[Kozerawski and Turk ()]{b22}\label{b22} 	 		\textit{One-Class Meta-Learning: Towards Generalizable Few-Shot Open-Set Classification},  		 			Jedrzej Kozerawski 		,  		 			Matthew Turk 		.  		 arXiv:2109.06859.  		2021.  	 	 (arXiv preprint) 
\bibitem[Huang ()]{b18}\label{b18} 	 		‘Orthogonal weight normalization: Solution to optimization over multiple dependent stiefel manifolds in deep neural networks’.  		 			Lei Huang 		.  	 	 		\textit{Thirty-Second AAAI Conference on Artificial Intelligence},  				2018.  	 
\bibitem[Zhou ()]{b35}\label{b35} 	 		‘Places:A10 million image database for scene recognition’.  		 			Bolei Zhou 		.  	 	 		\textit{IEEE transactions}  		2017. 40  (6)  p. .  	 
\bibitem[Walter et al. ()]{b30}\label{b30} 	 		‘Probability models for open set recognition’.  		 			J Walter 		,  		 			Scheirer 		,  		 			P Lalit 		,  		 			Terrance E Jain 		,  		 			Boult 		.  	 	 		\textit{IEEE transactions on pattern analysis and machine intelligence},  				2014. 36 p. .  	 
\bibitem[Akshay Raj Dhamija et al. ()]{b6}\label{b6} 	 		\textit{Reducing network agnostophobia},  		 			Manuel Akshay Raj Dhamija 		,  		 			Terrancee Günther 		,  		 			Boult 		.  		 arXiv:1811.04110.  		2018.  	 	 (arXiv preprint) 
\bibitem[Lu et al. ()]{b25}\label{b25} 	 		‘Safetynet: Detecting and rejecting adversarial examples robustly’.  		 			Jiajun Lu 		,  		 			Theerasit Issaranon 		,  		 			David Forsyth 		.  	 	 		\textit{Proceedings of the IEEE},  				 (the IEEE)  		2017. p. .  	 
\bibitem[Walter and Scheirer ()]{b31}\label{b31} 	 		‘Toward open set recognition’.  		 			J Walter 		,  		 			Scheirer 		.  	 	 		\textit{IEEE transactions on pattern analysis and machine intelligence},  				2012. 35 p. .  	 
\bibitem[Bendale and Boult ()]{b3}\label{b3} 	 		‘Towards open set deep networks’.  		 			Abhijit Bendale 		,  		 			Terrance E Boult 		.  	 	 		\textit{Proceedings of the IEEE conference on computer vision and pattern recognition},  				 (the IEEE conference on computer vision and pattern recognition)  		2016. p. .  	 
\bibitem[Simonyan and Zisserman ()]{b32}\label{b32} 	 		\textit{Very deep convolutional networks for large-scale image recognition},  		 			Karen Simonyan 		,  		 			Andrew Zisserman 		.  		 arXiv:1409.1556.  		2014.  	 	 (arXiv preprint) 
\bibitem[Wang ()]{b33}\label{b33} 	 		‘Video object discovery and cosegmentation with extremely weak supervision’.  		 			Le Wang 		.  	 	 		\textit{European Conference on Computer Vision},  				2014. Springer. p. .  	 
\end{bibitemlist}
 			 		 	 
\end{document}
