/*

 * jcdctmgr.c

 *

 * This file was part of the Independent JPEG Group's software:

 * Copyright (C) 1994-1996, Thomas G. Lane.

 * libjpeg-turbo Modifications:

 * Copyright (C) 1999-2006, MIYASAKA Masaru.

 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB</ossman@cendio.se>

 * Copyright (C) 2011, 2014-2015, D. R. Commander.

 * For conditions of distribution and use, see the accompanying README.ijg

 * file.

 *

 * This file contains the forward-DCT management logic.

 * This code selects a particular DCT implementation to be used,

 * and it performs related housekeeping chores including coefficient

 * quantization.

 */

#define JPEG_INTERNALS

#include "jinclude.h"

#include "jpeglib.h"

#include "jdct.h"               /* Private declarations for DCT subsystem */

#include "jsimddct.h"

/* Private subobject for this module */

typedef void (*forward_DCT_method_ptr) (DCTELEM *data);

typedef void (*float_DCT_method_ptr) (FAST_FLOAT *data);

typedef void (*convsamp_method_ptr) (JSAMPARRAY sample_data,

                                     JDIMENSION start_col,

                                     DCTELEM *workspace);

typedef void (*float_convsamp_method_ptr) (JSAMPARRAY sample_data,

                                           JDIMENSION start_col,

                                           FAST_FLOAT *workspace);

typedef void (*quantize_method_ptr) (JCOEFPTR coef_block, DCTELEM *divisors,

                                     DCTELEM *workspace);

typedef void (*float_quantize_method_ptr) (JCOEFPTR coef_block,

                                           FAST_FLOAT *divisors,

                                           FAST_FLOAT *workspace);

METHODDEF(void) quantize(JCOEFPTR, DCTELEM *, DCTELEM *);

typedef struct {

  struct jpeg_forward_dct pub;  /* public fields */

  /* Pointer to the DCT routine actually in use */

  forward_DCT_method_ptr dct;

  convsamp_method_ptr convsamp;

  quantize_method_ptr quantize;

  /* The actual post-DCT divisors --- not identical to the quant table

   * entries, because of scaling (especially for an unnormalized DCT).

   * Each table is given in normal array order.

   */

  DCTELEM *divisors[NUM_QUANT_TBLS];

  /* work area for FDCT subroutine */

  DCTELEM *workspace;

#ifdef DCT_FLOAT_SUPPORTED

  /* Same as above for the floating-point case. */

  float_DCT_method_ptr float_dct;

  float_convsamp_method_ptr float_convsamp;

  float_quantize_method_ptr float_quantize;

  FAST_FLOAT *float_divisors[NUM_QUANT_TBLS];

  FAST_FLOAT *float_workspace;

#endif

} my_fdct_controller;

typedef my_fdct_controller *my_fdct_ptr;

#if BITS_IN_JSAMPLE == 8

/*

 * Find the highest bit in an integer through binary search.

 */

LOCAL(int)

flss(UINT16 val)

{

  int bit;

  bit = 16;

  if (!val)

    return 0;

  if (!(val & 0xff00)) {

    bit -= 8;

    val <<= 8;

  }

  if (!(val & 0xf000)) {

    bit -= 4;

    val <<= 4;

  }

  if (!(val & 0xc000)) {

    bit -= 2;

    val <<= 2;

  }

  if (!(val & 0x8000)) {

    bit -= 1;

    val <<= 1;

  }

  return bit;

}

/*

 * Compute values to do a division using reciprocal.

 *

 * This implementation is based on an algorithm described in

 *   "How to optimize for the Pentium family of microprocessors"

 *   (http://www.agner.org/assem/).

 * More information about the basic algorithm can be found in

 * the paper "Integer Division Using Reciprocals" by Robert Alverson.

 *

 * The basic idea is to replace x/d by x * d^-1. In order to store

 * d^-1 with enough precision we shift it left a few places. It turns

 * out that this algoright gives just enough precision, and also fits

 * into DCTELEM:

 *

 *   b = (the number of significant bits in divisor) - 1

 *   r = (word size) + b

 *   f = 2^r / divisor

 *

 * f will not be an integer for most cases, so we need to compensate

 * for the rounding error introduced:

 *

 *   no fractional part:

 *

 *       result = input >> r

 *

 *   fractional part of f < 0.5:

 *

 *       round f down to nearest integer

 *       result = ((input + 1) * f) >> r

 *

 *   fractional part of f > 0.5:

 *

 *       round f up to nearest integer

 *       result = (input * f) >> r

 *

 * This is the original algorithm that gives truncated results. But we

 * want properly rounded results, so we replace "input" with

 * "input + divisor/2".

 *

 * In order to allow SIMD implementations we also tweak the values to

 * allow the same calculation to be made at all times:

 *

 *   dctbl[0] = f rounded to nearest integer

 *   dctbl[1] = divisor / 2 (+ 1 if fractional part of f < 0.5)

 *   dctbl[2] = 1 << ((word size) * 2 - r)

 *   dctbl[3] = r - (word size)

 *

 * dctbl[2] is for stupid instruction sets where the shift operation

 * isn't member wise (e.g. MMX).

 *

 * The reason dctbl[2] and dctbl[3] reduce the shift with (word size)

 * is that most SIMD implementations have a "multiply and store top

 * half" operation.

 *

 * Lastly, we store each of the values in their own table instead

 * of in a consecutive manner, yet again in order to allow SIMD

 * routines.

 */

LOCAL(int)

compute_reciprocal(UINT16 divisor, DCTELEM *dtbl)

{

  UDCTELEM2 fq, fr;

  UDCTELEM c;

  int b, r;

  if (divisor == 1) {

    /* divisor == 1 means unquantized, so these reciprocal/correction/shift

     * values will cause the C quantization algorithm to act like the

     * identity function.  Since only the C quantization algorithm is used in

     * these cases, the scale value is irrelevant.

     */

    dtbl[DCTSIZE2 * 0] = (DCTELEM)1;                        /* reciprocal */

    dtbl[DCTSIZE2 * 1] = (DCTELEM)0;                        /* correction */

    dtbl[DCTSIZE2 * 2] = (DCTELEM)1;                        /* scale */

    dtbl[DCTSIZE2 * 3] = -(DCTELEM)(sizeof(DCTELEM) * 8);   /* shift */

    return 0;

  }

  b = flss(divisor) - 1;

  r  = sizeof(DCTELEM) * 8 + b;

  fq = ((UDCTELEM2)1 << r) / divisor;

  fr = ((UDCTELEM2)1 << r) % divisor;

  c = divisor / 2;                      /* for rounding */

  if (fr == 0) {                        /* divisor is power of two */

    /* fq will be one bit too large to fit in DCTELEM, so adjust */

    fq >>= 1;

    r--;

  } else if (fr <= (divisor / 2U)) {    /* fractional part is < 0.5 */

    c++;

  } else {                              /* fractional part is > 0.5 */

    fq++;

  }

  dtbl[DCTSIZE2 * 0] = (DCTELEM)fq;     /* reciprocal */

  dtbl[DCTSIZE2 * 1] = (DCTELEM)c;      /* correction + roundfactor */

#ifdef WITH_SIMD

  dtbl[DCTSIZE2 * 2] = (DCTELEM)(1 << (sizeof(DCTELEM) * 8 * 2 - r)); /* scale */

#else

  dtbl[DCTSIZE2 * 2] = 1;

#endif

  dtbl[DCTSIZE2 * 3] = (DCTELEM)r - sizeof(DCTELEM) * 8; /* shift */

  if (r <= 16) return 0;

  else return 1;

}

#endif

/*

 * Initialize for a processing pass.

 * Verify that all referenced Q-tables are present, and set up

 * the divisor table for each one.

 * In the current implementation, DCT of all components is done during

 * the first pass, even if only some components will be output in the

 * first scan.  Hence all components should be examined here.

 */

METHODDEF(void)

start_pass_fdctmgr(j_compress_ptr cinfo)

{

  my_fdct_ptr fdct = (my_fdct_ptr)cinfo->fdct;

  int ci, qtblno, i;

  jpeg_component_info *compptr;

  JQUANT_TBL *qtbl;

  DCTELEM *dtbl;

  for (ci = 0, compptr = cinfo->comp_info; ci < cinfo->num_components;

       ci++, compptr++) {

    qtblno = compptr->quant_tbl_no;

    /* Make sure specified quantization table is present */

    if (qtblno < 0 || qtblno >= NUM_QUANT_TBLS ||

        cinfo->quant_tbl_ptrs[qtblno] == NULL)

      ERREXIT1(cinfo, JERR_NO_QUANT_TABLE, qtblno);

    qtbl = cinfo->quant_tbl_ptrs[qtblno];

    /* Compute divisors for this quant table */

    /* We may do this more than once for same table, but it's not a big deal */

    switch (cinfo->dct_method) {

#ifdef DCT_ISLOW_SUPPORTED

    case JDCT_ISLOW:

      /* For LL&M IDCT method, divisors are equal to raw quantization

       * coefficients multiplied by 8 (to counteract scaling).

       */

      if (fdct->divisors[qtblno] == NULL) {

        fdct->divisors[qtblno] = (DCTELEM *)

          (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,

                                      (DCTSIZE2 * 4) * sizeof(DCTELEM));

      }

      dtbl = fdct->divisors[qtblno];

      for (i = 0; i < DCTSIZE2; i++) {

#if BITS_IN_JSAMPLE == 8

        if (!compute_reciprocal(qtbl->quantval[i] << 3, &dtbl[i]) &&

            fdct->quantize == jsimd_quantize)

          fdct->quantize = quantize;

#else

        dtbl[i] = ((DCTELEM)qtbl->quantval[i]) << 3;

#endif

      }

      break;

#endif

#ifdef DCT_IFAST_SUPPORTED

    case JDCT_IFAST:

      {

        /* For AA&N IDCT method, divisors are equal to quantization

         * coefficients scaled by scalefactor[row]*scalefactor[col], where

         *   scalefactor[0] = 1

         *   scalefactor[k] = cos(k*PI/16) * sqrt(2)    for k=1..7

         * We apply a further scale factor of 8.

         */

#define CONST_BITS  14

        static const INT16 aanscales[DCTSIZE2] = {

          /* precomputed values scaled up by 14 bits */

          16384, 22725, 21407, 19266, 16384, 12873,  8867,  4520,

          22725, 31521, 29692, 26722, 22725, 17855, 12299,  6270,

          21407, 29692, 27969, 25172, 21407, 16819, 11585,  5906,

          19266, 26722, 25172, 22654, 19266, 15137, 10426,  5315,

          16384, 22725, 21407, 19266, 16384, 12873,  8867,  4520,

          12873, 17855, 16819, 15137, 12873, 10114,  6967,  3552,

           8867, 12299, 11585, 10426,  8867,  6967,  4799,  2446,

           4520,  6270,  5906,  5315,  4520,  3552,  2446,  1247

        };

        SHIFT_TEMPS

        if (fdct->divisors[qtblno] == NULL) {

          fdct->divisors[qtblno] = (DCTELEM *)

            (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,

                                        (DCTSIZE2 * 4) * sizeof(DCTELEM));

        }

        dtbl = fdct->divisors[qtblno];

        for (i = 0; i < DCTSIZE2; i++) {

#if BITS_IN_JSAMPLE == 8

          if (!compute_reciprocal(

                DESCALE(MULTIPLY16V16((JLONG)qtbl->quantval[i],

                                      (JLONG)aanscales[i]),

                        CONST_BITS - 3), &dtbl[i]) &&

              fdct->quantize == jsimd_quantize)

            fdct->quantize = quantize;

#else

          dtbl[i] = (DCTELEM)

            DESCALE(MULTIPLY16V16((JLONG)qtbl->quantval[i],

                                  (JLONG)aanscales[i]),

                    CONST_BITS - 3);

#endif

        }

      }

      break;

#endif

#ifdef DCT_FLOAT_SUPPORTED

    case JDCT_FLOAT:

      {

        /* For float AA&N IDCT method, divisors are equal to quantization

         * coefficients scaled by scalefactor[row]*scalefactor[col], where

         *   scalefactor[0] = 1

         *   scalefactor[k] = cos(k*PI/16) * sqrt(2)    for k=1..7

         * We apply a further scale factor of 8.

         * What's actually stored is 1/divisor so that the inner loop can

         * use a multiplication rather than a division.

         */

        FAST_FLOAT *fdtbl;

        int row, col;

        static const double aanscalefactor[DCTSIZE] = {

          1.0, 1.387039845, 1.306562965, 1.175875602,

          1.0, 0.785694958, 0.541196100, 0.275899379

        };

        if (fdct->float_divisors[qtblno] == NULL) {

          fdct->float_divisors[qtblno] = (FAST_FLOAT *)

            (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,

                                        DCTSIZE2 * sizeof(FAST_FLOAT));

        }

        fdtbl = fdct->float_divisors[qtblno];

        i = 0;

        for (row = 0; row < DCTSIZE; row++) {

          for (col = 0; col < DCTSIZE; col++) {

            fdtbl[i] = (FAST_FLOAT)

              (1.0 / (((double)qtbl->quantval[i] *

                       aanscalefactor[row] * aanscalefactor[col] * 8.0)));

            i++;

          }

        }

      }

      break;

#endif

    default:

      ERREXIT(cinfo, JERR_NOT_COMPILED);

      break;

    }

  }

}

/*

 * Load data into workspace, applying unsigned->signed conversion.

 */

METHODDEF(void)

convsamp(JSAMPARRAY sample_data, JDIMENSION start_col, DCTELEM *workspace)

{

  register DCTELEM *workspaceptr;

  register JSAMPROW elemptr;

  register int elemr;

  workspaceptr = workspace;

  for (elemr = 0; elemr < DCTSIZE; elemr++) {

    elemptr = sample_data[elemr] + start_col;

#if DCTSIZE == 8                /* unroll the inner loop */

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

#else

    {

      register int elemc;

      for (elemc = DCTSIZE; elemc > 0; elemc--)

        *workspaceptr++ = GETJSAMPLE(*elemptr++) - CENTERJSAMPLE;

    }

#endif

  }

}

/*

 * Quantize/descale the coefficients, and store into coef_blocks[].

 */

METHODDEF(void)

quantize(JCOEFPTR coef_block, DCTELEM *divisors, DCTELEM *workspace)

{

  int i;

  DCTELEM temp;

  JCOEFPTR output_ptr = coef_block;

#if BITS_IN_JSAMPLE == 8

  UDCTELEM recip, corr;

  int shift;

  UDCTELEM2 product;

  for (i = 0; i < DCTSIZE2; i++) {

    temp = workspace[i];

    recip = divisors[i + DCTSIZE2 * 0];

    corr =  divisors[i + DCTSIZE2 * 1];

    shift = divisors[i + DCTSIZE2 * 3];

    if (temp < 0) {

      temp = -temp;

      product = (UDCTELEM2)(temp + corr) * recip;

      product >>= shift + sizeof(DCTELEM) * 8;

      temp = (DCTELEM)product;

      temp = -temp;

    } else {

      product = (UDCTELEM2)(temp + corr) * recip;

      product >>= shift + sizeof(DCTELEM) * 8;

      temp = (DCTELEM)product;

    }

    output_ptr[i] = (JCOEF)temp;

  }

#else

  register DCTELEM qval;

  for (i = 0; i < DCTSIZE2; i++) {

    qval = divisors[i];

    temp = workspace[i];

    /* Divide the coefficient value by qval, ensuring proper rounding.

     * Since C does not specify the direction of rounding for negative

     * quotients, we have to force the dividend positive for portability.

     *

     * In most files, at least half of the output values will be zero

     * (at default quantization settings, more like three-quarters...)

     * so we should ensure that this case is fast.  On many machines,

     * a comparison is enough cheaper than a divide to make a special test

     * a win.  Since both inputs will be nonnegative, we need only test

     * for a < b to discover whether a/b is 0.

     * If your machine's division is fast enough, define FAST_DIVIDE.

     */

#ifdef FAST_DIVIDE

#define DIVIDE_BY(a, b)  a /= b

#else

#define DIVIDE_BY(a, b)  if (a >= b) a /= b;  else a = 0

#endif

    if (temp < 0) {

      temp = -temp;

      temp += qval >> 1;        /* for rounding */

      DIVIDE_BY(temp, qval);

      temp = -temp;

    } else {

      temp += qval >> 1;        /* for rounding */

      DIVIDE_BY(temp, qval);

    }

    output_ptr[i] = (JCOEF)temp;

  }

#endif

}

/*

 * Perform forward DCT on one or more blocks of a component.

 *

 * The input samples are taken from the sample_data[] array starting at

 * position start_row/start_col, and moving to the right for any additional

 * blocks. The quantized coefficients are returned in coef_blocks[].

 */

METHODDEF(void)

forward_DCT(j_compress_ptr cinfo, jpeg_component_info *compptr,

            JSAMPARRAY sample_data, JBLOCKROW coef_blocks,

            JDIMENSION start_row, JDIMENSION start_col, JDIMENSION num_blocks)

/* This version is used for integer DCT implementations. */

{

  /* This routine is heavily used, so it's worth coding it tightly. */

  my_fdct_ptr fdct = (my_fdct_ptr)cinfo->fdct;

  DCTELEM *divisors = fdct->divisors[compptr->quant_tbl_no];

  DCTELEM *workspace;

  JDIMENSION bi;

  /* Make sure the compiler doesn't look up these every pass */

  forward_DCT_method_ptr do_dct = fdct->dct;

  convsamp_method_ptr do_convsamp = fdct->convsamp;

  quantize_method_ptr do_quantize = fdct->quantize;

  workspace = fdct->workspace;

  sample_data += start_row;     /* fold in the vertical offset once */

  for (bi = 0; bi < num_blocks; bi++, start_col += DCTSIZE) {

    /* Load data into workspace, applying unsigned->signed conversion */

    (*do_convsamp) (sample_data, start_col, workspace);

    /* Perform the DCT */

    (*do_dct) (workspace);

    /* Quantize/descale the coefficients, and store into coef_blocks[] */

    (*do_quantize) (coef_blocks[bi], divisors, workspace);

  }

}

#ifdef DCT_FLOAT_SUPPORTED

METHODDEF(void)

convsamp_float(JSAMPARRAY sample_data, JDIMENSION start_col,

               FAST_FLOAT *workspace)

{

  register FAST_FLOAT *workspaceptr;

  register JSAMPROW elemptr;

  register int elemr;

  workspaceptr = workspace;

  for (elemr = 0; elemr < DCTSIZE; elemr++) {

    elemptr = sample_data[elemr] + start_col;

#if DCTSIZE == 8                /* unroll the inner loop */

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    *workspaceptr++ = (FAST_FLOAT)(GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

#else

    {

      register int elemc;

      for (elemc = DCTSIZE; elemc > 0; elemc--)

        *workspaceptr++ = (FAST_FLOAT)

                          (GETJSAMPLE(*elemptr++) - CENTERJSAMPLE);

    }

#endif

  }

}

METHODDEF(void)

quantize_float(JCOEFPTR coef_block, FAST_FLOAT *divisors,

               FAST_FLOAT *workspace)

{

  register FAST_FLOAT temp;

  register int i;

  register JCOEFPTR output_ptr = coef_block;

  for (i = 0; i < DCTSIZE2; i++) {

    /* Apply the quantization and scaling factor */

    temp = workspace[i] * divisors[i];

    /* Round to nearest integer.

     * Since C does not specify the direction of rounding for negative

     * quotients, we have to force the dividend positive for portability.

     * The maximum coefficient size is +-16K (for 12-bit data), so this

     * code should work for either 16-bit or 32-bit ints.

     */

    output_ptr[i] = (JCOEF)((int)(temp + (FAST_FLOAT)16384.5) - 16384);

  }

}

METHODDEF(void)

forward_DCT_float(j_compress_ptr cinfo, jpeg_component_info *compptr,

                  JSAMPARRAY sample_data, JBLOCKROW coef_blocks,

                  JDIMENSION start_row, JDIMENSION start_col,

                  JDIMENSION num_blocks)

/* This version is used for floating-point DCT implementations. */

{

  /* This routine is heavily used, so it's worth coding it tightly. */

  my_fdct_ptr fdct = (my_fdct_ptr)cinfo->fdct;

  FAST_FLOAT *divisors = fdct->float_divisors[compptr->quant_tbl_no];

  FAST_FLOAT *workspace;

  JDIMENSION bi;

  /* Make sure the compiler doesn't look up these every pass */

  float_DCT_method_ptr do_dct = fdct->float_dct;

  float_convsamp_method_ptr do_convsamp = fdct->float_convsamp;

  float_quantize_method_ptr do_quantize = fdct->float_quantize;

  workspace = fdct->float_workspace;

  sample_data += start_row;     /* fold in the vertical offset once */

  for (bi = 0; bi < num_blocks; bi++, start_col += DCTSIZE) {

    /* Load data into workspace, applying unsigned->signed conversion */

    (*do_convsamp) (sample_data, start_col, workspace);

    /* Perform the DCT */

    (*do_dct) (workspace);

    /* Quantize/descale the coefficients, and store into coef_blocks[] */

    (*do_quantize) (coef_blocks[bi], divisors, workspace);

  }

}

#endif /* DCT_FLOAT_SUPPORTED */

/*

 * Initialize FDCT manager.

 */

GLOBAL(void)

jinit_forward_dct(j_compress_ptr cinfo)

{

  my_fdct_ptr fdct;

  int i;

  fdct = (my_fdct_ptr)

    (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,

                                sizeof(my_fdct_controller));

  cinfo->fdct = (struct jpeg_forward_dct *)fdct;

  fdct->pub.start_pass = start_pass_fdctmgr;

  /* First determine the DCT... */

  switch (cinfo->dct_method) {

#ifdef DCT_ISLOW_SUPPORTED

  case JDCT_ISLOW:

    fdct->pub.forward_DCT = forward_DCT;

    if (jsimd_can_fdct_islow())

      fdct->dct = jsimd_fdct_islow;

    else

      fdct->dct = jpeg_fdct_islow;

    break;

#endif

#ifdef DCT_IFAST_SUPPORTED

  case JDCT_IFAST:

    fdct->pub.forward_DCT = forward_DCT;

    if (jsimd_can_fdct_ifast())

      fdct->dct = jsimd_fdct_ifast;

    else

      fdct->dct = jpeg_fdct_ifast;

    break;

#endif

#ifdef DCT_FLOAT_SUPPORTED

  case JDCT_FLOAT:

    fdct->pub.forward_DCT = forward_DCT_float;

    if (jsimd_can_fdct_float())

      fdct->float_dct = jsimd_fdct_float;

    else

      fdct->float_dct = jpeg_fdct_float;

    break;

#endif

  default:

    ERREXIT(cinfo, JERR_NOT_COMPILED);

    break;

  }

  /* ...then the supporting stages. */

  switch (cinfo->dct_method) {

#ifdef DCT_ISLOW_SUPPORTED

  case JDCT_ISLOW:

#endif

#ifdef DCT_IFAST_SUPPORTED

  case JDCT_IFAST:

#endif

#if defined(DCT_ISLOW_SUPPORTED) || defined(DCT_IFAST_SUPPORTED)

    if (jsimd_can_convsamp())

      fdct->convsamp = jsimd_convsamp;

    else

      fdct->convsamp = convsamp;

    if (jsimd_can_quantize())

      fdct->quantize = jsimd_quantize;

    else

      fdct->quantize = quantize;

    break;

#endif

#ifdef DCT_FLOAT_SUPPORTED

  case JDCT_FLOAT:

    if (jsimd_can_convsamp_float())

      fdct->float_convsamp = jsimd_convsamp_float;

    else

      fdct->float_convsamp = convsamp_float;

    if (jsimd_can_quantize_float())

      fdct->float_quantize = jsimd_quantize_float;

    else

      fdct->float_quantize = quantize_float;

    break;

#endif

  default:

    ERREXIT(cinfo, JERR_NOT_COMPILED);

    break;

  }

  /* Allocate workspace memory */

#ifdef DCT_FLOAT_SUPPORTED

  if (cinfo->dct_method == JDCT_FLOAT)

    fdct->float_workspace = (FAST_FLOAT *)

      (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,

                                  sizeof(FAST_FLOAT) * DCTSIZE2);

  else

#endif

    fdct->workspace = (DCTELEM *)

      (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,

                                  sizeof(DCTELEM) * DCTSIZE2);

  /* Mark divisor tables unallocated */

  for (i = 0; i < NUM_QUANT_TBLS; i++) {

    fdct->divisors[i] = NULL;

#ifdef DCT_FLOAT_SUPPORTED

    fdct->float_divisors[i] = NULL;

#endif

  }

}