<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Archiving and Interchange DTD v2.3 20070202//EN" "archivearticle.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" article-type="methods-article" dtd-version="2.3" xml:lang="EN">
<front>
<journal-meta>
<journal-id journal-id-type="publisher-id">Front. Immunol.</journal-id>
<journal-title>Frontiers in Immunology</journal-title>
<abbrev-journal-title abbrev-type="pubmed">Front. Immunol.</abbrev-journal-title>
<issn pub-type="epub">1664-3224</issn>
<publisher>
<publisher-name>Frontiers Media S.A.</publisher-name>
</publisher>
</journal-meta>
<article-meta>
<article-id pub-id-type="doi">10.3389/fimmu.2021.681636</article-id>
<article-categories>
<subj-group subj-group-type="heading">
<subject>Immunology</subject>
<subj-group>
<subject>Methods</subject>
</subj-group>
</subj-group>
</article-categories>
<title-group>
<article-title>AutoPlate: Rapid Dose-Response Curve Analysis for Biological Assays</article-title>
</title-group>
<contrib-group>
<contrib contrib-type="author" corresp="yes">
<name>
<surname>Palmer</surname><given-names>Phil</given-names>
</name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="author-notes" rid="fn001"><sup>*</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/1264241"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Del Rosario</surname><given-names>Joanne Marie M.</given-names>
</name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<xref ref-type="aff" rid="aff3"><sup>3</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/921979"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>da Costa</surname><given-names>Kelly A. S.</given-names>
</name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/1219697"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Carnell</surname><given-names>George W.</given-names>
</name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/194858"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Huang</surname><given-names>Chloe Q.</given-names>
</name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Heeney</surname><given-names>Jonathan L.</given-names>
</name>
<xref ref-type="aff" rid="aff1"><sup>1</sup></xref>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/229649"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Temperton</surname><given-names>Nigel J.</given-names>
</name>
<xref ref-type="aff" rid="aff2"><sup>2</sup></xref>
<uri xlink:href="https://loop.frontiersin.org/people/185243"/>
</contrib>
<contrib contrib-type="author">
<name>
<surname>Wells</surname><given-names>David A.</given-names>
</name>
<xref ref-type="aff" rid="aff4"><sup>4</sup></xref>
</contrib>
</contrib-group>
<aff id="aff1"><sup>1</sup><institution>Laboratory of Viral Zoonotics, Department of Veterinary Medicine, University of Cambridge</institution>, <addr-line>Cambridge</addr-line>, <country>United Kingdom</country></aff>
<aff id="aff2"><sup>2</sup><institution>Viral Pseudotype Unit, Medway School of Pharmacy, University of Kent</institution>, <addr-line>Chatham</addr-line>, <country>United Kingdom</country></aff>
<aff id="aff3"><sup>3</sup><institution>Department of Physical Sciences and Mathematics, College of Arts and Sciences, University of the Philippines Manila</institution>, <addr-line>Manila</addr-line>, <country>Philippines</country></aff>
<aff id="aff4"><sup>4</sup><institution>DIOSynVax, University of Cambridge</institution>, <addr-line>Cambridge</addr-line>, <country>United Kingdom</country></aff>
<author-notes>
<fn fn-type="edited-by">
<p>Edited by: Luis Eduardo Coelho Andrade, Federal University of S&#xe3;o Paulo, Brazil</p>
</fn>
<fn fn-type="edited-by">
<p>Reviewed by: Wesley Prieto, Fleury S.A., Brazil; Anatoly Sorokin, University of Liverpool, United Kingdom</p>
</fn>
<fn fn-type="corresp" id="fn001">
<p>*Correspondence: Phil Palmer, <email xlink:href="mailto:pp502@cam.ac.uk">pp502@cam.ac.uk</email></p>
</fn>
<fn fn-type="other" id="fn002">
<p>This article was submitted to Viral Immunology, a section of the journal Frontiers in Immunology</p>
</fn>
</author-notes>
<pub-date pub-type="epub">
<day>10</day>
<month>02</month>
<year>2022</year>
</pub-date>
<pub-date pub-type="collection">
<year>2021</year>
</pub-date>
<volume>12</volume>
<elocation-id>681636</elocation-id>
<history>
<date date-type="received">
<day>16</day>
<month>03</month>
<year>2021</year>
</date>
<date date-type="accepted">
<day>29</day>
<month>12</month>
<year>2021</year>
</date>
</history>
<permissions>
<copyright-statement>Copyright &#xa9; 2022 Palmer, Del Rosario, da Costa, Carnell, Huang, Heeney, Temperton and Wells</copyright-statement>
<copyright-year>2022</copyright-year>
<copyright-holder>Palmer, Del Rosario, da Costa, Carnell, Huang, Heeney, Temperton and Wells</copyright-holder>
<license xlink:href="http://creativecommons.org/licenses/by/4.0/">
<p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (CC BY). The use, distribution or reproduction in other forums is permitted, provided the original author(s) and the copyright owner(s) are credited and that the original publication in this journal is cited, in accordance with accepted academic practice. No use, distribution or reproduction is permitted which does not comply with these terms.</p>
</license>
</permissions>
<abstract>
<p>The emergence of COVID-19 has emphasised that biological assay data must be analysed quickly to develop safe, effective and timely vaccines/therapeutics. For viruses such as SARS-CoV-2, the primary way of measuring immune correlates of protection is through assays such as the pseudotype microneutralisation (pMN) assay, thanks to its safety and versatility. However, despite the presence of existing tools for data analysis such as PRISM and R the analysis of these assays remains cumbersome and time-consuming. We introduce an open-source R Shiny web application and R library (AutoPlate) to accelerate data analysis of dose-response curve immunoassays. Using example data from influenza studies, we show that AutoPlate improves on available analysis software in terms of ease of use, flexibility and speed. AutoPlate (<uri xlink:href="https://philpalmer.shinyapps.io/AutoPlate/">https://philpalmer.shinyapps.io/AutoPlate/</uri>) is a tool for the use of laboratories and wider scientific community to accelerate the analysis of biological assays in the development of viral vaccines and therapeutics.</p>
</abstract>
<kwd-group>
<kwd>pseudotype neutralization</kwd>
<kwd>ELLA</kwd>
<kwd>data analysis</kwd>
<kwd>biological assay</kwd>
<kwd>dose-response</kwd>
<kwd>pseudovirus</kwd>
<kwd>SARS-C0V-2</kwd>
<kwd>influenza</kwd>
</kwd-group>
<contract-sponsor id="cn001">Innovate UK<named-content content-type="fundref-id">10.13039/501100006041</named-content>
</contract-sponsor>
<contract-sponsor id="cn002">UK Research and Innovation<named-content content-type="fundref-id">10.13039/100014013</named-content>
</contract-sponsor>
<contract-sponsor id="cn003">Open Philanthropy Project<named-content content-type="fundref-id">10.13039/100014895</named-content>
</contract-sponsor>
<counts>
<fig-count count="10"/>
<table-count count="1"/>
<equation-count count="2"/>
<ref-count count="30"/>
<page-count count="10"/>
<word-count count="3921"/>
</counts>
</article-meta>
</front>
<body>
<sec id="s1">
<title>Introduction</title>
<p>The pseudotype based microneutralisation (pMN) assay measures functional antibody responses against viruses (<xref ref-type="bibr" rid="B1">1</xref>). The pMN assay uses pseudotypes or pseudoviruses, viral vectors which usually display the envelope protein of the virus of interest on its surface with a marker or reporter gene, commonly luciferase or green fluorescent protein (<xref ref-type="bibr" rid="B2">2</xref>, <xref ref-type="bibr" rid="B3">3</xref>). The viral entry protein or envelope enables the pseudotypes to enter cells which then express the encoded marker allowing viral entry to be quantified. Antibodies that inhibit virus entry will reduce the expression of the pseudotype marker and so the potency of neutralisation can also be quantified (<xref ref-type="bibr" rid="B1">1</xref>, <xref ref-type="bibr" rid="B4">4</xref>).</p>
<p>Neutralisation potency is usually reported as the IC<sub>50</sub> of a dose-response curve. By measuring neutralisation along a dilution series, a dose-response curve can be estimated for a given antibody/sera (<xref ref-type="bibr" rid="B1">1</xref>). When this dilution is displayed on a log scale the curve follows a classic S shape which is well represented by the 4-parameter log-logistic regression curve (<xref ref-type="bibr" rid="B5">5</xref>). As shown by <xref ref-type="fig" rid="f1"><bold>Figure 1</bold></xref>, the IC<sub>50</sub> is defined as the dilution which gives 50% neutralisation of the curve (<xref ref-type="bibr" rid="B1">1</xref>, <xref ref-type="bibr" rid="B5">5</xref>).</p>
<fig id="f1" position="float">
<label>Figure 1</label>
<caption>
<p>Schematic representation of the pseudotype neutralisation (pMN) assay. Steps 1-5 show the lab protocol for the pMN assay and steps 6-8 show the computational analysis using AutoPlate. The plate layout for the well types can be seen where the sera (type &#x201c;x&#x201d;) are shown in yellow, antibody positive control (type &#x201c;m&#x201d;) shown in red, virus (type &#x201c;v&#x201d;) shown in blue and cells (type &#x201c;c&#x201d;) shown in pink (created with BioRender) (<xref ref-type="bibr" rid="B6">6</xref>).</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g001.tif"/>
</fig>
<p>Influenza pseudotypes are predominantly used for the pMN assays but can also be used for the enzyme-linked lectin assay (ELLA) that measure inhibition of neuraminidase (NA) activity (<xref ref-type="bibr" rid="B7">7</xref>). Neuraminidase is the second most abundant influenza surface glycoprotein after haemagglutinin (HA). ELLA can also be performed using 96-well plates in almost the same format as pMN, with the response shown <italic>via</italic> dose-response curves and reported as IC<sub>50</sub> values (<xref ref-type="bibr" rid="B8">8</xref>).</p>
<p>There are three major advantages of pMN and pELLA (pseudotype based ELLA) compared to other biological assays (bioassays) for measuring immune response against viruses. These assays are very safe (<xref ref-type="bibr" rid="B1">1</xref>, <xref ref-type="bibr" rid="B2">2</xref>, <xref ref-type="bibr" rid="B9">9</xref>), versatile (<xref ref-type="bibr" rid="B2">2</xref>, <xref ref-type="bibr" rid="B3">3</xref>), as they can be used for a range of viruses, and have growing adoption for emerging viruses (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B10">10</xref>, <xref ref-type="bibr" rid="B11">11</xref>).</p>
<p>The assays are safe because the pseudotypes used are replication-incompetent meaning that they cannot replicate as they do not contain all the genes from the original viral vector (most commonly a lentivirus or retrovirus) needed to replicate (<xref ref-type="bibr" rid="B1">1</xref>, <xref ref-type="bibr" rid="B2">2</xref>). As a result, these assays can be performed at a lower biosafety level (BSL) (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B9">9</xref>, <xref ref-type="bibr" rid="B11">11</xref>). For example, SARS-CoV-2 pMN can be performed in BSL 2 laboratories but live SARS-CoV-2 requires BSL 3 facilities, further increasing the speed at which vaccines and other therapeutics can be developed (<xref ref-type="bibr" rid="B4">4</xref>, <xref ref-type="bibr" rid="B9">9</xref>, <xref ref-type="bibr" rid="B12">12</xref>).</p>
<p>The pMN assay can be applied to virtually any enveloped virus as it measures cell entry rather than a specific feature of the virus (<xref ref-type="bibr" rid="B2">2</xref>). pMN has been applied to many viruses including influenza (<xref ref-type="bibr" rid="B1">1</xref>, <xref ref-type="bibr" rid="B12">12</xref>&#x2013;<xref ref-type="bibr" rid="B15">15</xref>), HIV (<xref ref-type="bibr" rid="B16">16</xref>, <xref ref-type="bibr" rid="B17">17</xref>), Ebola (<xref ref-type="bibr" rid="B18">18</xref>, <xref ref-type="bibr" rid="B19">19</xref>), MERS (<xref ref-type="bibr" rid="B9">9</xref>, <xref ref-type="bibr" rid="B20">20</xref>), Dengue (<xref ref-type="bibr" rid="B21">21</xref>), Lassa (<xref ref-type="bibr" rid="B22">22</xref>), Rabies (<xref ref-type="bibr" rid="B23">23</xref>), Chikungunya (<xref ref-type="bibr" rid="B24">24</xref>) and Nipah virus (<xref ref-type="bibr" rid="B25">25</xref>). It has become one of the principal assays for characterising functional immune response during the ongoing SARS-CoV-2 pandemic (<xref ref-type="bibr" rid="B4">4</xref>, <xref ref-type="bibr" rid="B12">12</xref>, <xref ref-type="bibr" rid="B26">26</xref>), which further indicates its rapid uptake and applicability to new and emerging viruses (<xref ref-type="bibr" rid="B3">3</xref>, <xref ref-type="bibr" rid="B10">10</xref>).</p>
<p>Once the experiment has been run the two main steps to analyze it are reformatting the data and statistical analysis (<xref ref-type="bibr" rid="B1">1</xref>). Although there are proprietary and open-source tools for the analysis there are drawbacks to currently available software solutions and the time-consuming reformatting is not handled by either.</p>
<p>The main input for the computational analysis of the immunoassays is raw luminescence (or fluorescence) data, often contained within tabular files (normally CSV or Excel) that specify relative luminescence units (RLU) values for each well (<xref ref-type="bibr" rid="B1">1</xref>). However, the crucial experimental metadata is usually not included and so must be carefully entered for each well. Along with reformatting the data to be entered into the chosen stats package, this is the most time-consuming step of the computational analysis and where an intuitive and efficient interface could most benefit labs running these assays.</p>
</sec>
<sec id="s2">
<title>Results</title>
<sec id="s2_1">
<title>AutoPlate</title>
<p>We present AutoPlate as a simple interface to quickly add experimental metadata to immunoassay results, reformat data and perform statistical analysis. AutoPlate produces publication-ready figures but allows users to export data for further analysis with external statistical software such as R. AutoPlate can be accessed through an online Shiny app or installed as an R package. The AutoPlate source code is open source and available at <uri xlink:href="https://github.com/PhilPalmer/AutoPlate">https://github.com/PhilPalmer/AutoPlate</uri>.</p>
</sec>
<sec id="s2_2">
<title>How Does AutoPlate Compare to Other Existing Software?</title>
<p>Existing proprietary software such as PRISM allows for the analysis of bioassays <italic>via</italic> a graphical user interface (GUI) (<xref ref-type="bibr" rid="B1">1</xref>). This helps make it easier to enter data, however, it is rigid compared to tools such as the open-source R and Python programming languages and there is little/no integration with these languages. The R and Python programming languages have software packages &#x201c;drc&#x201d; and &#x201c;neutcurve&#x201d; respectively (<xref ref-type="bibr" rid="B5">5</xref>, <xref ref-type="bibr" rid="B27">27</xref>). These packages are incredibly flexible for dose-response curve analysis but require a technical understanding of their respective programming languages (<xref ref-type="bibr" rid="B5">5</xref>). Crucially, preparing data for analysis is slow in all programs especially when analysing many 96-well plates, as shown in <xref ref-type="table" rid="T1"><bold>Table 1</bold></xref>.</p>
<table-wrap id="T1" position="float">
<label>Table 1</label>
<caption>
<p>Qualitative comparison between AutoPlate and currently available software for analysing data from bioassays.</p>
</caption>
<table frame="hsides">
<thead>
<tr>
<th valign="top" align="left">Tool</th>
<th valign="top" align="center">Graphical user interface (GUI) available? (Ease of Use)</th>
<th valign="top" align="center">Command line software package available? (Flexibility)</th>
<th valign="top" align="center">Handles reformatting of raw plate data? (Data Entry Speed)</th>
</tr>
</thead>
<tbody>
<tr>
<td valign="top" align="left">AutoPlate</td>
<td valign="top" align="left" style="background-color:#92d050">Yes</td>
<td valign="top" align="left" style="background-color:#92d050">Yes</td>
<td valign="top" align="left" style="background-color:#92d050">Yes</td>
</tr>
<tr>
<td valign="top" align="left">PRISM</td>
<td valign="top" align="left" style="background-color:#92d050">Yes</td>
<td valign="top" align="left" style="background-color:#ff5959">No</td>
<td valign="top" align="left" style="background-color:#ff5959">No</td>
</tr>
<tr>
<td valign="top" align="left">R (drc)</td>
<td valign="top" align="left" style="background-color:#ff5959">No</td>
<td valign="top" align="left" style="background-color:#92d050">Yes</td>
<td valign="top" align="left" style="background-color:#ff5959">No</td>
</tr>
<tr>
<td valign="top" align="left">Python (neutcurve)</td>
<td valign="top" align="left" style="background-color:#ff5959">No</td>
<td valign="top" align="left" style="background-color:#92d050">Yes</td>
<td valign="top" align="left" style="background-color:#ff5959">No</td>
</tr>
</tbody>
</table>
</table-wrap>
</sec>
<sec id="s2_3">
<title>Overview of the Application</title>
<p>AutoPlate provides an intuitive graphical user interface for quickly adding experimental metadata and formatting the data for analysis. This formatted data can be exported for analysis in other software or analysed within AutoPlate to produce a report including quality control checks and publication-quality figures. AutoPlate is implemented in the R Shiny framework (<uri xlink:href="https://shiny.rstudio.com/">https://shiny.rstudio.com/</uri>) because it combines the flexibility of R (with packages such as &#x201c;drc&#x201d;) with a user-friendly graphical user interface (similar to PRISM). Analysing data using AutoPlate follows three steps.</p>
<sec id="s2_3_1">
<title>Step 1) Input</title>
<p>To add experimental metadata before data analysis AutoPlate accepts five inputs (see <xref ref-type="fig" rid="f2"><bold>Figure 2</bold></xref>):</p>
<list list-type="simple">
<list-item>
<p>1) The assay type is defined, currently supported assays are the pMN and ELLA assays.</p>
</list-item>
<list-item>
<p>2) Luminescence files that contain the raw luminescence values for each well can be uploaded directly from the plate reader. CSV format is used for the pMN assay and Excel format for the ELLA assay. Users can also upload a previously exported CSV file generated from a later step in AutoPlate if they want to re-load a dataset that has already been analysed.</p>
</list-item>
<list-item>
<p>3) Both the serum and control concentration/dilutions can be specified by an interactive table, containing default values (<xref ref-type="fig" rid="f3"><bold>Figure 3</bold></xref>). Each row in the table corresponds to rows or columns on the 96-well plate for the pMN and ELLA assays respectively.</p>
</list-item>
<list-item>
<p>4) For each uploaded 96-well plate, an 8x12 interactive table will be generated. This table can be used to view and change the values for each well for any feature such as the sample types, treatment group or sample ID, etc., by selecting that feature from the drop-down menu. Initially, wells are populated with default values commonly used in our lab for the types, sample IDs and dilutions. For example, for the pMN assay, two columns are used per sample with the sample ID increasing across the plate(s), with the first and last columns being reserved for the controls. Modifications can be made in bulk by modifying the &#x201c;template&#x201d; to allow any plate layout to be quickly propagated across all plates.</p>
</list-item>
</list>
<fig id="f2" position="float">
<label>Figure 2</label>
<caption>
<p>AutoPlate Step 1) Input Screenshot. Upload raw data from the plate reader and specify metadata required for dose-response curve analysis.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g002.tif"/>
</fig>
<fig id="f3" position="float">
<label>Figure 3</label>
<caption>
<p>AutoPlate Step 1) Input Concentrations/Dilutions Table Screenshot. Specify the serum and control concentrations/dilutions.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g003.tif"/>
</fig>
<p>The default pMN types layout can be seen in <xref ref-type="fig" rid="f4"><bold>Figure 4</bold></xref>. Most wells are type &#x201c;x&#x201d; (experimental); these are a known dilution of sample neutralizing the virus of interest. Type &#x201c;m&#x201d; indicates the positive control (&#x201c;m&#x201d; was originally for monoclonal antibodies), these wells are treated the same as type &#x201c;x&#x201d; but can be easily filtered or colored differently and may be diluted differently. The final types are &#x201c;v&#x201d; and &#x201c;c&#x201d; which are used to convert virus marker measurements into neutralisation and normalize the data. Type &#x201c;c&#x201d; contains cells only and represents 100% neutralisation because there is no virus to enter cells and express the marker. Type &#x201c;v&#x201d; contains cells and virus-only but (no treatment) and represents 0% neutralisation because no virus neutralisation occurs.</p>
<fig id="f4" position="float">
<label>Figure 4</label>
<caption>
<p>AutoPlate Step 1) Plate Data Table Screenshot. Enter data for any plate, well or feature such as the well type.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g004.tif"/>
</fig>
<list list-type="simple">
<list-item>
<p>5) The final input is any other features such as the bleed, treatment, virus and experiment ID. The bleed is normally an integer corresponding to the week the sample (e.g., mouse) was bled or &#x201c;terminal&#x201d; for the last bleed. The treatment will be used to group the data for the dose-response curves, for example, what the subjects were inoculated with. The virus refers to the pseudotyped virus (pseudovirus) that was used in the assay. The experiment ID is a unique identifier specifying which study generated the data. You can have multiple experiment IDs if you are analysing multiple datasets from different experiments. All these features can be set using the 96-well plate data tables or in the &#x201c;other features&#x201d; section. This implementation of the other features section was designed to be as flexible as possible, allowing the user to set these features by selecting any existing feature and providing a mapping for a new value. For example, it is often best to set the treatment based upon the sample ID if you are specifying which vaccination each subject received. These four features were chosen as default because they are common to our group. However, these features could be omitted, and other ones added in the future.</p>
</list-item>
</list>
</sec>
<sec id="s2_3_2">
<title>Step 2) Quality Control</title>
<p>The quality control step (<xref ref-type="fig" rid="f5"><bold>Figure 5</bold></xref>) allows users to quickly determine that the data is entered correctly and that the controls have worked as expected. To visualize the data entered in Step 1&#xa0;a table, various heatmaps and a boxplot are generated. If the controls for a particular plate or well have failed, then these wells can be excluded from the analysis.</p>
<fig id="f5" position="float">
<label>Figure 5</label>
<caption>
<p>AutoPlate Step 2) Quality Control Screenshot. Visualise the data entered in Step 1 and check that the controls have worked for each plate/well.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g005.tif"/>
</fig>
<p>A table is generated (<xref ref-type="fig" rid="f6"><bold>Figure 6</bold></xref>) displaying the average viral and cell luminescence to check that there is a substantial difference in the average luminescence between virus-only and cell-only wells and that an adequate number of control wells have been included in the analysis.</p>
<fig id="f6" position="float">
<label>Figure 6</label>
<caption>
<p>AutoPlate Step 2) Quality Control Average Luminescence Table Screenshot. Check the number of control wells and difference in luminescence values between the virus-only and cell-only control wells.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g006.tif"/>
</fig>
<p>Heatmaps are generated for various key features (well types, sample ID, dilution, virus, RLU, neutralisation (<xref ref-type="fig" rid="f7"><bold>Figure 7</bold></xref>), treatment, bleed and experiment ID). For all these features a heatmap is used to show the value for each of these features for all plates in a 96-well plate format i.e., an 8 x 12 grid.</p>
<fig id="f7" position="float">
<label>Figure 7</label>
<caption>
<p>AutoPlate Step 2) Quality Control Neutralisation Heatmap Screenshot. Visualise features such as the neutralisation in a 96-well plate format using a heatmap.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g007.tif"/>
</fig>
<p>A user can choose to exclude any wells from the analysis. To do this they can specify a string consisting of comma-separated values of either whole plates, individual wells or a range of wells. Wells can also be excluded from the plate data table in the input step (<xref ref-type="fig" rid="f4"><bold>Figure 4</bold></xref>). Wells could be excluded for any number of reasons but are most commonly excluded when the controls have failed or when wells were left empty on a particular plate. A heatmap is also generated to show which wells will be excluded from the analysis, allowing users to verify that the correct wells have been excluded from the analysis.</p>
<p>A boxplot of all the different well types for each plate is generated (<xref ref-type="fig" rid="f8"><bold>Figure 8</bold></xref>). This plot can be used to check differences in control values between plates. It is important to check that the virus wells have approximately the highest luminescence and cell wells have the lowest because it is these wells that are used for the normalization when calculating the neutralisation. In our experience, if the virus-only wells do not have the highest values on a plate, the dose-response curves estimated from that plate are shifted downwards, and this may underestimate the neutralizing capacity of the sample sera.</p>
<fig id="f8" position="float">
<label>Figure 8</label>
<caption>
<p>AutoPlate Step 2) Quality Control Types Boxplot Screenshot. The graph shows the average raw luminescence value for each well type on each plate. The legend functions as a toggle to show which well types are shown.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g008.tif"/>
</fig>
<p>Users can export the formatted dataset containing all assay data as a CSV from AutoPlate. This allows the data to be shared or analysed with different statistical software. This dataset can also be uploaded on the first step of AutoPlate so that shared data can be analysed in AutoPlate or old data can be reanalyzed.</p>
</sec>
<sec id="s2_3_3">
<title>Step 3) Results</title>
<p>The following four interactive plots are generated in the results section (<xref ref-type="fig" rid="f9"><bold>Figure 9</bold></xref>), data exploration, dose-response curve, IC<sub>50</sub> boxplot and virus-cell boxplot. All these plots can be downloaded as a publication-quality figure in either SVG or PNG format and the raw code required to reproduce the plot in R can be viewed.</p>
<fig id="f9" position="float">
<label>Figure 9</label>
<caption>
<p>AutoPlate Step 3) Results Screenshot. Fit dose-response curves to analyse the data entered in previous steps, calculate the IC<sub>50</sub> values and generate downloadable plots to visualise the results.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g009.tif"/>
</fig>
<p>The data exploration plot (<xref ref-type="fig" rid="f10"><bold>Figure 10A</bold></xref>) fits a loess smooth to each treatment group to illustrate the dose-response relationship of treatment groups outside of the dose-response curve model. The virus-cell boxplot can be used to check that the controls have worked as expected (<xref ref-type="fig" rid="f10"><bold>Figure 10B</bold></xref>). There should be a clear separation between the virus and cell groups and little variation between plates. The dose-response tab fits a 4-parameter log-logistic regression dose-response model to each of the treatments (<xref ref-type="fig" rid="f10"><bold>Figure 10C</bold></xref>) (<xref ref-type="bibr" rid="B5">5</xref>). To simplify visual comparison of treatments a boxplot of just the IC<sub>50</sub> values of each curve is displayed (<xref ref-type="fig" rid="f10"><bold>Figure 10D</bold></xref>).</p>
<fig id="f10" position="float">
<label>Figure 10</label>
<caption>
<p>AutoPlate Step 3) Results Plots Screenshot <bold>(A)</bold> Data Exploration Plot. For each virus the serum dilution is plotted on the x-axis against the neutralisation on the y-axis. Each line represents a different treatment group. <bold>(B)</bold> Virus Cell Boxplot. The graph shows cell and virus only well types on the x-axis for each plate plotted against the log raw luminescence value. <bold>(C)</bold> Dose-response Curves Plot. The plot for virus A/Michigan/45/2015 (H1N1) is shown with the serum dilution plotted on the x-axis against the neutralisation on the y-axis. Each line represents a single sample and is coloured by the treatment group. <bold>(D)</bold> IC<sub>50</sub> Boxplot. The plot for virus A/Michigan/45/2015 (H1N1) is shown with the log IC<sub>50</sub> dilution plotted on the x-axis against the treatment group on the y-axis. Each data point represents a single sample.</p>
</caption>
<graphic mimetype="image" mime-subtype="tiff" xlink:href="fimmu-12-681636-g010.tif"/>
</fig>
<p>The dose-response model, which is used for the dose-response curve and IC<sub>50</sub> box plot, can also be defined, allowing a great deal of flexibility. To produce the plots, the &#x201c;drc&#x201d; R package is used and the user can define any parameters to the &#x201c;drm&#x201d; function they wish such as the equation, model type, which parameters are shared between treatment groups, and any upper/lower limits for the parameters used by the dose-response model (<xref ref-type="bibr" rid="B5">5</xref>).</p>
<p>All the information displayed within AutoPlate, such as the quality control and results plots can be downloaded as a shareable HTML report file. As well as including all the plots generated in AutoPlate, it also contains the specific code and R package versions needed to reproduce them.</p>
</sec>
</sec>
<sec id="s2_4">
<title>Applications of AutoPlate</title>
<p>To show the speed, flexibility and ease of use of AutoPlate we have provided an example dataset. This dataset was used for all figures, can be loaded within AutoPlate and the raw data can be found within the GitHub link. The dataset consists of five different Influenza H1N1 viral pseudotypes tested against mice inoculated with PBS, A/Michigan/45/2015 (MI/15) and A/Brisbane/02/2018 (Bris/18). The dataset shows a range of neutralisation responses, as shown in <xref ref-type="fig" rid="f10"><bold>Figure 10</bold></xref>. For all viruses, the negative control (PBS) shows very low neutralisation and the positive control (HoxB8 Antibody, 4F8) shows very high neutralisation. The two inoculates show very low neutralisation except when they are homologous i.e. the strain against which the mice were inoculated is the same as the viral pseudotype, in which case the mice showed moderate neutralisation.</p>
</sec>
</sec>
<sec id="s3">
<title>Discussion</title>
<p>AutoPlate is a tool accessible through a web application or R package, to help further automate the analysis of bioassays. The main objective of AutoPlate was to enable users to go from raw data to publication-quality plots quickly by handling the time-consuming reformatting of data which is not handled by the other tools. This allows considerable time saving and therefore shortens the feedback loop for data analysis, which is especially useful when conducting large high throughput experiments.</p>
<p>We also aimed to make AutoPlate user-friendly. There is no need for the user to have previous programming experience or any sophisticated software installed on their computer, only a web browser, unlike software packages drc and neutcurve. This helps make the data analysis more accessible to the scientist performing the assay, who may have crucial input or valuable domain-specific knowledge.</p>
<p>Golem is a useful tool for developing Shiny applications with software best practices as a standalone R package so that users can run their own version (<xref ref-type="bibr" rid="B28">28</xref>). AutoPlate development was supported with Golem including automated testing, extensive documentation and ensuring the package consists mainly of modular R functions. This makes AutoPlate easier to maintain, install and means that users can use AutoPlate as a package rather than a Shiny app.</p>
<p>Finally, AutoPlate is very flexible compared tools such as PRISM, due to its integration with R. It is possible to install AutoPlate as an R library, view the raw R code for all the plots and get a list of all the R packages used including versions from the HTML report. This makes reproducing the analysis within R trivial, allowing the user to tweak or extend the analysis performed with all the options available in R. Any analysis within the AutoPlate web application is reproducible because it is possible to export the full dataset and re-uploading and analysing the same dataset will generate the same results. As AutoPlate is open-source, it is also highly extendable. For example, AutoPlate could support new analyses, new assay types such as ELISA (enzyme-linked immunosorbent assay), HIA (hemagglutination inhibition assay) or even any custom assay and other plate types such as 384-well plates.</p>
<p>In summary, AutoPlate is a fast, easy to use and flexible web application to help accelerate the analysis of biological assay data.</p>
</sec>
<sec id="s4">
<title>Methods</title>
<p>AutoPlate was developed using R v3.6.3 and R Shiny v1.5.0 (<xref ref-type="bibr" rid="B29">29</xref>, <xref ref-type="bibr" rid="B30">30</xref>). The neutralisation is calculated from the RLU values and normalized per plate. The minimum level of cell infection is calculated as the median luminescence from the virus-only wells, for a given plate. The maximum level of cell infection is calculated as the median luminescence from the cell-only wells, for a given plate. The neutralisation for each well is then expressed as a percentage between the minimum and maximum levels of cell infection as calculated for that plate. This is shown in the equation below for calculating the neutralisation, where x is the RLU for a particular well, v is the median RLU for the virus-only wells for the plate and c&#xa0;is the median RLU for the cell-only wells for the same plate.</p>
<disp-formula>
<mml:math display="block" id="M1">
<mml:mrow>
<mml:mi>n</mml:mi>
<mml:mi>e</mml:mi>
<mml:mi>u</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>r</mml:mi>
<mml:mi>a</mml:mi>
<mml:mi>l</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>s</mml:mi>
<mml:mi>a</mml:mi>
<mml:mi>t</mml:mi>
<mml:mi>i</mml:mi>
<mml:mi>o</mml:mi>
<mml:mi>n</mml:mi>
<mml:mo>=</mml:mo>
<mml:mn>100</mml:mn>
<mml:mo>&#xd7;</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mi>x</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>v</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mi>c</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>v</mml:mi>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
</disp-formula>
<p>Dose-response curves and IC<sub>50</sub> values were estimated using the R package drc v3.0-1 (<xref ref-type="bibr" rid="B5">5</xref>). By default, for the dose-response model, the dose is the dilution, and the neutralisation is the response, the sample IDs are used to group the data. The model used is the four-parameter log-logistic function (or LL2.4) which can be denoted by the expression below, where b is the slope around the IC<sub>50</sub>, c is the curve minimum, d is the curve maximum and e is the log IC<sub>50</sub> value (<xref ref-type="bibr" rid="B5">5</xref>). This model was chosen as the default option because we found that models more readily converge when estimating log(IC<sub>50</sub>) than IC<sub>50</sub> itself. By default, a single curve minimum, curve maximum, and gradient are estimated for the whole population to improve comparisons between samples. The upper limit of the curve maximum is set to 100 and the lower limit for the gradient around the IC<sub>50</sub> and the IC<sub>50</sub> value are set to zero to prevent a negative gradient or IC<sub>50</sub> value.</p>
<disp-formula>
<mml:math display="block" id="M3">
<mml:mrow>
<mml:mi>f</mml:mi>
<mml:mo stretchy="false">(</mml:mo>
<mml:mi>x</mml:mi>
<mml:mo stretchy="false">)</mml:mo>
<mml:mo>=</mml:mo>
<mml:mi>c</mml:mi>
<mml:mo>+</mml:mo>
<mml:mfrac>
<mml:mrow>
<mml:mi>d</mml:mi>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>c</mml:mi>
</mml:mrow>
<mml:mrow>
<mml:mn>1</mml:mn>
<mml:mo>+</mml:mo>
<mml:mi>exp</mml:mi>
<mml:mo stretchy="false">(</mml:mo>
<mml:mi>b</mml:mi>
<mml:mo stretchy="false">(</mml:mo>
<mml:mi>log</mml:mi>
<mml:mo stretchy="false">(</mml:mo>
<mml:mi>x</mml:mi>
<mml:mo stretchy="false">)</mml:mo>
<mml:mo>&#x2212;</mml:mo>
<mml:mi>e</mml:mi>
<mml:mo stretchy="false">)</mml:mo>
<mml:mo stretchy="false">)</mml:mo>
</mml:mrow>
</mml:mfrac>
</mml:mrow>
</mml:math>
</disp-formula>
</sec>
<sec id="s5">
<title>Code Availability</title>
<p>The AutoPlate web application can be accessed freely at <uri xlink:href="https://philpalmer.shinyapps.io/AutoPlate/">https://philpalmer.shinyapps.io/AutoPlate/</uri>. Users are encouraged to modify, contribute or deploy their own version of AutoPlate. The documentation is available at <uri xlink:href="https://philpalmer.github.io/AutoPlate/">https://philpalmer.github.io/AutoPlate/</uri> and the source code at <uri xlink:href="https://github.com/PhilPalmer/AutoPlate">https://github.com/PhilPalmer/AutoPlate</uri>.</p>
</sec>
<sec id="s6" sec-type="data-availability">
<title>Data Availability Statement</title>
<p>The datasets presented in this study can be found in online repositories. The names of the repository/repositories and accession number(s) can be found below: <uri xlink:href="https://github.com/PhilPalmer/AutoPlate">https://github.com/PhilPalmer/AutoPlate</uri>.</p>
</sec>
<sec id="s7" sec-type="author-contributions">
<title>Author Contributions</title>
<p>The project was conceived by DW who also provided the initial code. This was further developed into a web application by PP with feedback from DW, JR, KC, and GC. CH generated the example pMN data for Influenza H1N1 which was provided by GC. DW, JH, and NT supervised the project. PP wrote the manuscript with support from all authors. All&#xa0;authors contributed to the article and approved the submitted version.</p>
</sec>
<sec id="s8" sec-type="funding-information">
<title>Funding</title>
<p>This work was supported in part by Innovate-UK PIVA grant, ref 105078 and the UKRI/NIHR grant nr. COV0170, the Humoral Immune Correlates to COVID-19 (HICC) consortium. PP is supported by a fellowship provided by The Open Philanthropy Project.</p>
</sec>
<sec id="s9" sec-type="COI-statement">
<title>Conflict of Interest</title>
<p>Authors DAW and JLH were employed by the company DIOSynVax.</p>
<p>The remaining authors declare that the research was conducted in the absence of any commercial or financial relationships that could be construed as a potential conflict of interest.</p>
</sec>
<sec id="s10" sec-type="disclaimer">
<title>Publisher&#x2019;s Note</title>
<p>All claims expressed in this article are solely those of the authors and do not necessarily represent those of their affiliated organizations, or those of the publisher, the editors and the reviewers. Any product that may be evaluated in this article, or claim that may be made by its manufacturer, is not guaranteed or endorsed by the publisher.</p>
</sec>
</body>
<back>
<sec sec-type="supplementary-material" id="s11">
<title>Supplementary Material</title>
<p>The Supplementary Material for this article can be found online at: <ext-link ext-link-type="uri" xlink:href="https://www.frontiersin.org/articles/10.3389/fimmu.2021.681636/full#supplementary-material">https://www.frontiersin.org/articles/10.3389/fimmu.2021.681636/full#supplementary-material</ext-link></p>
<supplementary-material xlink:href="DataSheet_1.docx" id="SM1" mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document"/>
</sec>
<ref-list>
<title>References</title>
<ref id="B1">
<label>1</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ferrara</surname> <given-names>F</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>N</given-names>
</name>
</person-group>. <article-title>Pseudotype Neutralization Assays: From Laboratory Bench to Data Analysis</article-title>. <source>Methods Protoc</source> (<year>2018</year>) <volume>1</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>15</lpage>. doi: <pub-id pub-id-type="doi">10.3390/mps1010008</pub-id>
</citation>
</ref>
<ref id="B2">
<label>2</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>King</surname> <given-names>B</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>NJ</given-names>
</name>
<name>
<surname>Grehan</surname> <given-names>K</given-names>
</name>
<name>
<surname>Scott</surname> <given-names>SD</given-names>
</name>
<name>
<surname>Wright</surname> <given-names>E</given-names>
</name>
<name>
<surname>Tarr</surname> <given-names>AW</given-names>
</name>
<etal/>
</person-group>. <article-title>Technical Considerations for the Generation of Novel Pseudotyped Viruses</article-title>. <source>Future Virol</source> (<year>2015</year>) <volume>11</volume>(<issue>1</issue>):<fpage>47</fpage>&#x2013;<lpage>59</lpage>. doi: <pub-id pub-id-type="doi">10.2217/fvl.15.106</pub-id>
</citation>
</ref>
<ref id="B3">
<label>3</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Li</surname> <given-names>Q</given-names>
</name>
<name>
<surname>Liu</surname> <given-names>Q</given-names>
</name>
<name>
<surname>Huang</surname> <given-names>W</given-names>
</name>
<name>
<surname>Li</surname> <given-names>X</given-names>
</name>
<name>
<surname>Wang</surname> <given-names>Y</given-names>
</name>
</person-group>. <article-title>Current Status on the Development of Pseudoviruses for Enveloped Viruses</article-title>. <source>Rev Med Virol</source> (<year>2018</year>) <volume>28</volume>(<issue>1</issue>):<elocation-id>e1963</elocation-id>. doi: <pub-id pub-id-type="doi">10.1002/rmv.1963</pub-id>
</citation>
</ref>
<ref id="B4">
<label>4</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Nie</surname> <given-names>J</given-names>
</name>
<name>
<surname>Li</surname> <given-names>Q</given-names>
</name>
<name>
<surname>Wu</surname> <given-names>J</given-names>
</name>
<name>
<surname>Zhao</surname> <given-names>C</given-names>
</name>
<name>
<surname>Hao</surname> <given-names>H</given-names>
</name>
<name>
<surname>Liu</surname> <given-names>H</given-names>
</name>
<etal/>
</person-group>. <article-title>Quantification of SARS-CoV-2 Neutralizing Antibody by a Pseudotyped Virus-Based Assay</article-title>. <source>Nat Protoc</source> (<year>2020</year>) <volume>15</volume>(<issue>11</issue>):<page-range>3699&#x2013;715</page-range>. doi: <pub-id pub-id-type="doi">10.1038/s41596-020-0394-5</pub-id>
</citation>
</ref>
<ref id="B5">
<label>5</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ritz</surname> <given-names>C</given-names>
</name>
<name>
<surname>Baty</surname> <given-names>F</given-names>
</name>
<name>
<surname>Streibig</surname> <given-names>JC</given-names>
</name>
<name>
<surname>Gerhard</surname> <given-names>D</given-names>
</name>
</person-group>. <article-title>Dose-Response Analysis Using R</article-title>. <source>PloS One</source> (<year>2015</year>) <volume>10</volume>(<issue>12</issue>):<elocation-id>e0146021</elocation-id>. doi: <pub-id pub-id-type="doi">10.1371/journal.pone.0146021</pub-id>
</citation>
</ref>
<ref id="B6">
<label>6</label>
<citation citation-type="web">
<source>BioRender.</source> Available at: <uri xlink:href="https://biorender.com/">https://biorender.com/</uri>.
</citation>
</ref>
<ref id="B7">
<label>7</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Biuso</surname> <given-names>F</given-names>
</name>
<name>
<surname>Carnell</surname> <given-names>G</given-names>
</name>
<name>
<surname>Montomoli</surname> <given-names>E</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>N</given-names>
</name>
</person-group>. <article-title>A Lentiviral Pseudotype ELLA for the Measurement of Antibodies Against Influenza Neuraminidase</article-title>. <source>Bio-Protoc</source> (<year>2018</year>) <volume>11</volume>:<page-range>e2936&#x2013;6</page-range>. doi: <pub-id pub-id-type="doi">10.21769/BioProtoc.2936</pub-id>
</citation>
</ref>
<ref id="B8">
<label>8</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Gao</surname> <given-names>J</given-names>
</name>
<name>
<surname>Couzens</surname> <given-names>L</given-names>
</name>
<name>
<surname>Eichelberger</surname> <given-names>MC</given-names>
</name>
</person-group>. <article-title>Measuring Influenza Neuraminidase Inhibition Antibody Titers by Enzyme-Linked Lectin Assay</article-title>. <source>J Vis Exp JoVE</source> (<year>2016</year>) <volume>115</volume>:<fpage>1</fpage>&#x2013;<lpage>4</lpage>. doi: <pub-id pub-id-type="doi">10.3791/54573</pub-id>
</citation>
</ref>
<ref id="B9">
<label>9</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Millet</surname> <given-names>JK</given-names>
</name>
<name>
<surname>Tang</surname> <given-names>T</given-names>
</name>
<name>
<surname>Nathan</surname> <given-names>L</given-names>
</name>
<name>
<surname>Jaimes</surname> <given-names>JA</given-names>
</name>
<name>
<surname>Hsu</surname> <given-names>H-L</given-names>
</name>
<name>
<surname>Daniel</surname> <given-names>S</given-names>
</name>
<etal/>
</person-group>. <article-title>Production of Pseudotyped Particles to Study Highly Pathogenic Coronaviruses in a Biosafety Level 2 Setting</article-title>. <source>JoVE J Vis Exp</source> (<year>2019</year>) <volume>145)</volume>:<elocation-id>e59010</elocation-id>. doi: <pub-id pub-id-type="doi">10.3791/59010</pub-id>
</citation>
</ref>
<ref id="B10">
<label>10</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Steffen</surname> <given-names>I</given-names>
</name>
<name>
<surname>Simmons</surname> <given-names>G</given-names>
</name>
</person-group>. <article-title>Pseudotyping Viral Vectors With Emerging Virus Envelope Proteins</article-title>. <source>Curr Gene Ther</source> (<year>2016</year>) <volume>16</volume>(<issue>1</issue>):<fpage>47</fpage>&#x2013;<lpage>55</lpage>. doi: <pub-id pub-id-type="doi">10.2174/1566523216666160119093948</pub-id>
</citation>
</ref>
<ref id="B11">
<label>11</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Bentley</surname> <given-names>E</given-names>
</name>
<name>
<surname>Mather</surname> <given-names>S</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>N</given-names>
</name>
</person-group>. <article-title>The Use of Pseudotypes to Study Viruses, Virus Sero-Epidemiology and Vaccination</article-title>. <source>Vaccine</source> (<year>2015</year>) <volume>33</volume>:<page-range>2955&#x2013;8</page-range>. doi: <pub-id pub-id-type="doi">10.1016/j.vaccine.2015.04.071</pub-id>
</citation>
</ref>
<ref id="B12">
<label>12</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Nie</surname> <given-names>J</given-names>
</name>
<name>
<surname>Li</surname> <given-names>Q</given-names>
</name>
<name>
<surname>Wu</surname> <given-names>J</given-names>
</name>
<name>
<surname>Zhao</surname> <given-names>C</given-names>
</name>
<name>
<surname>Hao</surname> <given-names>H</given-names>
</name>
<name>
<surname>Liu</surname> <given-names>H</given-names>
</name>
<etal/>
</person-group>. <article-title>Establishment and Validation of a Pseudovirus Neutralization Assay for SARS-CoV-2</article-title>. <source>Emerg Microbes Infect</source> (<year>2020</year>) <volume>9</volume>(<issue>1</issue>):<page-range>680&#x2013;6</page-range>. doi: <pub-id pub-id-type="doi">10.1080/22221751.2020.1743767</pub-id>
</citation>
</ref>
<ref id="B13">
<label>13</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Carnell</surname> <given-names>GW</given-names>
</name>
<name>
<surname>Ferrara</surname> <given-names>F</given-names>
</name>
<name>
<surname>Grehan</surname> <given-names>K</given-names>
</name>
<name>
<surname>Thompson</surname> <given-names>CP</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>NJ</given-names>
</name>
</person-group>. <article-title>Pseudotype-Based Neutralization Assays for Influenza: A Systematic Analysis</article-title>. <source>Front Immunol</source> (<year>2015</year>) <volume>6</volume>:<elocation-id>161</elocation-id>. doi:&#xa0;<pub-id pub-id-type="doi">10.3389/fimmu.2015.00161/full</pub-id>
</citation>
</ref>
<ref id="B14">
<label>14</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ao</surname> <given-names>Z</given-names>
</name>
<name>
<surname>Patel</surname> <given-names>A</given-names>
</name>
<name>
<surname>Tran</surname> <given-names>K</given-names>
</name>
<name>
<surname>He</surname> <given-names>X</given-names>
</name>
<name>
<surname>Fowke</surname> <given-names>K</given-names>
</name>
<name>
<surname>Coombs</surname> <given-names>K</given-names>
</name>
<etal/>
</person-group>. <article-title>Characterization of a Trypsin-Dependent Avian Influenza H5N1-Pseudotyped HIV Vector System for High Throughput Screening of Inhibitory Molecules</article-title>. <source>Antiviral Res</source> (<year>2008</year>) <volume>79</volume>(<issue>1</issue>):<page-range>12&#x2013;8</page-range>. doi: <pub-id pub-id-type="doi">10.1016/j.antiviral.2008.02.001</pub-id>
</citation>
</ref>
<ref id="B15">
<label>15</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Wallerstr&#xf6;m</surname> <given-names>S</given-names>
</name>
<name>
<surname>Lagerqvist</surname> <given-names>N</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>NJ</given-names>
</name>
<name>
<surname>Cassmer</surname> <given-names>M</given-names>
</name>
<name>
<surname>Moreno</surname> <given-names>A</given-names>
</name>
<name>
<surname>Karlsson</surname> <given-names>M</given-names>
</name>
<etal/>
</person-group>. <article-title>Detection of Antibodies Against H5 and H7 Strains in Birds: Evaluation of Influenza Pseudovirus Particle Neutralization Tests</article-title>. <source>Infect Ecol Epidemiol</source> (<year>2014</year>) <volume>4</volume>:<fpage>1</fpage>&#x2013;<lpage>5</lpage>. doi: <pub-id pub-id-type="doi">10.3402/iee.v4.23011</pub-id>
</citation>
</ref>
<ref id="B16">
<label>16</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Kim</surname> <given-names>YB</given-names>
</name>
<name>
<surname>Lee</surname> <given-names>MK</given-names>
</name>
<name>
<surname>Han</surname> <given-names>DP</given-names>
</name>
<name>
<surname>Cho</surname> <given-names>MW</given-names>
</name>
</person-group>. <article-title>Development of a Safe and Rapid Neutralization Assay Using Murine Leukemia Virus Pseudotyped With HIV Type 1 Envelope Glycoprotein Lacking the Cytoplasmic Domain</article-title>. <source>AIDS Res Hum Retroviruses</source> (<year>2001</year>) <volume>17</volume>(<issue>18</issue>):<page-range>1715&#x2013;24</page-range>. doi: <pub-id pub-id-type="doi">10.1089/08892220152741414</pub-id>
</citation>
</ref>
<ref id="B17">
<label>17</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Chan</surname> <given-names>E</given-names>
</name>
<name>
<surname>Heilek-Snyder</surname> <given-names>G</given-names>
</name>
<name>
<surname>Cammack</surname> <given-names>N</given-names>
</name>
<name>
<surname>Sankuratri</surname> <given-names>S</given-names>
</name>
<name>
<surname>Ji</surname> <given-names>C</given-names>
</name>
</person-group>. <article-title>Development of a Moloney Murine Leukemia Virus-Based Pseudotype Anti-HIV Assay Suitable for Accurate and Rapid Evaluation of HIV Entry Inhibitors</article-title>. <source>J Biomol Screen</source> (<year>2006</year>) <volume>11</volume>(<issue>6</issue>):<page-range>652&#x2013;63</page-range>. doi: <pub-id pub-id-type="doi">10.1177/1087057106288881</pub-id>
</citation>
</ref>
<ref id="B18">
<label>18</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Kobinger</surname> <given-names>GP</given-names>
</name>
<name>
<surname>Weiner</surname> <given-names>DJ</given-names>
</name>
<name>
<surname>Yu</surname> <given-names>Q-C</given-names>
</name>
<name>
<surname>Wilson</surname> <given-names>JM</given-names>
</name>
</person-group>. <article-title>Filovirus -Pseudotyped Lentiviral Vector can Efficiently and Stably Transduce Airway Epithelia In Vivo</article-title>. <source>Nat Biotechnol</source> (<year>2001</year>) <volume>19</volume>(<issue>3</issue>):<page-range>225&#x2013;30</page-range>. doi: <pub-id pub-id-type="doi">10.1038/85664</pub-id>
</citation>
</ref>
<ref id="B19">
<label>19</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Quinn</surname> <given-names>K</given-names>
</name>
<name>
<surname>Brindley</surname> <given-names>MA</given-names>
</name>
<name>
<surname>Weller</surname> <given-names>ML</given-names>
</name>
<name>
<surname>Kaludov</surname> <given-names>N</given-names>
</name>
<name>
<surname>Kondratowicz</surname> <given-names>A</given-names>
</name>
<name>
<surname>Hunt</surname> <given-names>CL</given-names>
</name>
<etal/>
</person-group>. <article-title>Rho GTPases Modulate Entry of Ebola Virus and Vesicular Stomatitis Virus Pseudotyped Vectors</article-title>. <source>J Virol</source> (<year>2009</year>) <volume>83</volume>(<issue>19</issue>):<page-range>10176&#x2013;86</page-range>. doi: <pub-id pub-id-type="doi">10.1128/JVI.00422-09</pub-id>
</citation>
</ref>
<ref id="B20">
<label>20</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Grehan</surname> <given-names>K</given-names>
</name>
<name>
<surname>Ferrara</surname> <given-names>F</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>N</given-names>
</name>
</person-group>. <article-title>An Optimised Method for the Production of MERS-CoV Spike Expressing Viral Pseudotypes</article-title>. <source>MethodsX</source> (<year>2015</year>) <volume>2</volume>:<page-range>379&#x2013;84</page-range>. doi: <pub-id pub-id-type="doi">10.1016/j.mex.2015.09.003</pub-id>
</citation>
</ref>
<ref id="B21">
<label>21</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Ansarah-Sobrinho</surname> <given-names>C</given-names>
</name>
<name>
<surname>Nelson</surname> <given-names>S</given-names>
</name>
<name>
<surname>Jost</surname> <given-names>CA</given-names>
</name>
<name>
<surname>Whitehead</surname> <given-names>SS</given-names>
</name>
<name>
<surname>Pierson</surname> <given-names>TC</given-names>
</name>
</person-group>. <article-title>Temperature-Dependent Production of Pseudoinfectious Dengue Reporter Virus Particles by Complementation</article-title>. <source>Virology</source> (<year>2008</year>) <volume>381</volume>(<issue>1</issue>):<fpage>67</fpage>&#x2013;<lpage>74</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.virol.2008.08.021</pub-id>
</citation>
</ref>
<ref id="B22">
<label>22</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Robinson</surname> <given-names>JE</given-names>
</name>
<name>
<surname>Hastie</surname> <given-names>KM</given-names>
</name>
<name>
<surname>Cross</surname> <given-names>RW</given-names>
</name>
<name>
<surname>Yenni</surname> <given-names>RE</given-names>
</name>
<name>
<surname>Elliott</surname> <given-names>DH</given-names>
</name>
<name>
<surname>Rouelle</surname> <given-names>JA</given-names>
</name>
<etal/>
</person-group>. <article-title>Most Neutralizing Human Monoclonal Antibodies Target Novel Epitopes Requiring Both Lassa Virus Glycoprotein Subunits</article-title>. <source>Nat Commun</source> (<year>2016</year>) <volume>7</volume>(<issue>1</issue>):<elocation-id>11544</elocation-id>. doi: <pub-id pub-id-type="doi">10.1038/ncomms11544</pub-id>
</citation>
</ref>
<ref id="B23">
<label>23</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Nie</surname> <given-names>J</given-names>
</name>
<name>
<surname>Wu</surname> <given-names>X</given-names>
</name>
<name>
<surname>Ma</surname> <given-names>J</given-names>
</name>
<name>
<surname>Cao</surname> <given-names>S</given-names>
</name>
<name>
<surname>Huang</surname> <given-names>W</given-names>
</name>
<name>
<surname>Liu</surname> <given-names>Q</given-names>
</name>
<etal/>
</person-group>. <article-title>Development of <italic>In Vitro</italic> and <italic>In Vivo</italic> Rabies Virus Neutralization Assays Based on a High-Titer Pseudovirus System</article-title>. <source>Sci Rep</source> (<year>2017</year>) <volume>7</volume>:<fpage>1</fpage>&#x2013;<lpage>10</lpage>. doi: <pub-id pub-id-type="doi">10.1038/srep42769</pub-id>
</citation>
</ref>
<ref id="B24">
<label>24</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Kishishita</surname> <given-names>N</given-names>
</name>
<name>
<surname>Takeda</surname> <given-names>N</given-names>
</name>
<name>
<surname>Anuegoonpipat</surname> <given-names>A</given-names>
</name>
<name>
<surname>Anantapreecha</surname> <given-names>S</given-names>
</name>
</person-group>. <article-title>Development of a Pseudotyped-Lentiviral-Vector-Based Neutralization Assay for Chikungunya Virus Infection</article-title>. <source>J Clin Microbiol</source> (<year>2013</year>) <volume>51</volume>(<issue>5</issue>):<page-range>1389&#x2013;95</page-range>. doi: <pub-id pub-id-type="doi">10.1128/JCM.03109-12</pub-id>
</citation>
</ref>
<ref id="B25">
<label>25</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Tamin</surname> <given-names>A</given-names>
</name>
<name>
<surname>Harcourt</surname> <given-names>BH</given-names>
</name>
<name>
<surname>Lo</surname> <given-names>MK</given-names>
</name>
<name>
<surname>Roth</surname> <given-names>JA</given-names>
</name>
<name>
<surname>Wolf</surname> <given-names>MC</given-names>
</name>
<name>
<surname>Lee</surname> <given-names>B</given-names>
</name>
<etal/>
</person-group>. <article-title>Development of a Neutralization Assay for Nipah Virus Using Pseudotype Particles</article-title>. <source>J Virol Methods</source> (<year>2009</year>) <volume>160</volume>(<issue>1</issue>):<fpage>1</fpage>&#x2013;<lpage>6</lpage>. doi: <pub-id pub-id-type="doi">10.1016/j.jviromet.2009.02.025</pub-id>
</citation>
</ref>
<ref id="B26">
<label>26</label>
<citation citation-type="journal">
<person-group person-group-type="author">
<name>
<surname>Hyseni</surname> <given-names>I</given-names>
</name>
<name>
<surname>Molesti</surname> <given-names>E</given-names>
</name>
<name>
<surname>Benincasa</surname> <given-names>L</given-names>
</name>
<name>
<surname>Piu</surname> <given-names>P</given-names>
</name>
<name>
<surname>Casa</surname> <given-names>E</given-names>
</name>
<name>
<surname>Temperton</surname> <given-names>NJ</given-names>
</name>
<etal/>
</person-group>. <article-title>Characterisation of SARS-CoV-2 Lentiviral Pseudotypes and Correlation Between Pseudotype-Based Neutralisation Assays and Live Virus-Based Micro Neutralisation Assays</article-title>. <source>Viruses</source> (<year>2020</year>) <volume>12</volume>(<issue>9</issue>):<fpage>1</fpage>&#x2013;<lpage>15</lpage>. doi: <pub-id pub-id-type="doi">10.3390/v12091011</pub-id>
</citation>
</ref>
<ref id="B27">
<label>27</label>
<citation citation-type="web">
<source>Neutcurve Documentation &#x2014; Neutcurve 0.5.1 Documentation.</source> Available at: <uri xlink:href="https://jbloomlab.github.io/neutcurve/">https://jbloomlab.github.io/neutcurve/</uri>.
</citation>
</ref>
<ref id="B28">
<label>28</label>
<citation citation-type="web">
<person-group person-group-type="author">
<name>
<surname>Guyader</surname> <given-names>V</given-names>
</name>
<name>
<surname>Fay</surname> <given-names>C</given-names>
</name>
<name>
<surname>Rochette</surname> <given-names>S</given-names>
</name>
<name>
<surname>Girard</surname> <given-names>C</given-names>
</name>
<name>
<surname>Nakov</surname> <given-names>N</given-names>
</name>
<name>
<surname>Think</surname> <given-names>R</given-names>
</name>
</person-group>. <source>Golem: A Framework for Robust Shiny Applications</source> (<year>2020</year>). Available at: <uri xlink:href="https://CRAN.R-project.org/package=golem">https://CRAN.R-project.org/package=golem</uri>.
</citation>
</ref>
<ref id="B29">
<label>29</label>
<citation citation-type="web">
<source>R: The R Project for Statistical Computing.</source> Available at: <uri xlink:href="https://www.r-project.org/">https://www.r-project.org/</uri>.
</citation>
</ref>
<ref id="B30">
<label>30</label>
<citation citation-type="web">
<person-group person-group-type="author">
<name>
<surname>Chang</surname> <given-names>W</given-names>
</name>
<name>
<surname>Cheng</surname> <given-names>J</given-names>
</name>
<name>
<surname>Allaire</surname> <given-names>JJ</given-names>
</name>
<name>
<surname>Xie</surname> <given-names>Y</given-names>
</name>
<name>
<surname>McPherson</surname> <given-names>J</given-names>
</name>
<collab>RStudio</collab>
<etal/>
</person-group>. <source>Shiny: Web Application Framework for R</source> (<year>2020</year>). Available at: <uri xlink:href="https://CRAN.R-project.org/package=shiny">https://CRAN.R-project.org/package=shiny</uri>.
</citation>
</ref>
</ref-list>
</back>
</article>