arch/mips/mach-octeon/include/mach/cvmx-fpa3.h - filogic/uboot - Gitiles

 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2020 Marvell International Ltd.
  *
  * Interface to the CN78XX Free Pool Allocator, a.k.a. FPA3
  */

 #include "cvmx-address.h"
 #include "cvmx-fpa-defs.h"
 #include "cvmx-scratch.h"

 #ifndef __CVMX_FPA3_H__
 #define __CVMX_FPA3_H__

 typedef struct {
 	unsigned res0 : 6;
 	unsigned node : 2;
 	unsigned res1 : 2;
 	unsigned lpool : 6;
 	unsigned valid_magic : 16;
 } cvmx_fpa3_pool_t;

 typedef struct {
 	unsigned res0 : 6;
 	unsigned node : 2;
 	unsigned res1 : 6;
 	unsigned laura : 10;
 	unsigned valid_magic : 16;
 } cvmx_fpa3_gaura_t;

 #define CVMX_FPA3_VALID_MAGIC	0xf9a3
 #define CVMX_FPA3_INVALID_GAURA ((cvmx_fpa3_gaura_t){ 0, 0, 0, 0, 0 })
 #define CVMX_FPA3_INVALID_POOL	((cvmx_fpa3_pool_t){ 0, 0, 0, 0, 0 })

 static inline bool __cvmx_fpa3_aura_valid(cvmx_fpa3_gaura_t aura)
 {
 	if (aura.valid_magic != CVMX_FPA3_VALID_MAGIC)
 		return false;
 	return true;
 }

 static inline bool __cvmx_fpa3_pool_valid(cvmx_fpa3_pool_t pool)
 {
 	if (pool.valid_magic != CVMX_FPA3_VALID_MAGIC)
 		return false;
 	return true;
 }

 static inline cvmx_fpa3_gaura_t __cvmx_fpa3_gaura(int node, int laura)
 {
 	cvmx_fpa3_gaura_t aura;

 	if (node < 0)
 		node = cvmx_get_node_num();
 	if (laura < 0)
 		return CVMX_FPA3_INVALID_GAURA;

 	aura.node = node;
 	aura.laura = laura;
 	aura.valid_magic = CVMX_FPA3_VALID_MAGIC;
 	return aura;
 }

 static inline cvmx_fpa3_pool_t __cvmx_fpa3_pool(int node, int lpool)
 {
 	cvmx_fpa3_pool_t pool;

 	if (node < 0)
 		node = cvmx_get_node_num();
 	if (lpool < 0)
 		return CVMX_FPA3_INVALID_POOL;

 	pool.node = node;
 	pool.lpool = lpool;
 	pool.valid_magic = CVMX_FPA3_VALID_MAGIC;
 	return pool;
 }

 #undef CVMX_FPA3_VALID_MAGIC

 /**
  * Structure describing the data format used for stores to the FPA.
  */
 typedef union {
 	u64 u64;
 	struct {
 		u64 scraddr : 8;
 		u64 len : 8;
 		u64 did : 8;
 		u64 addr : 40;
 	} s;
 	struct {
 		u64 scraddr : 8;
 		u64 len : 8;
 		u64 did : 8;
 		u64 node : 4;
 		u64 red : 1;
 		u64 reserved2 : 9;
 		u64 aura : 10;
 		u64 reserved3 : 16;
 	} cn78xx;
 } cvmx_fpa3_iobdma_data_t;

 /**
  * Struct describing load allocate operation addresses for FPA pool.
  */
 union cvmx_fpa3_load_data {
 	u64 u64;
 	struct {
 		u64 seg : 2;
 		u64 reserved1 : 13;
 		u64 io : 1;
 		u64 did : 8;
 		u64 node : 4;
 		u64 red : 1;
 		u64 reserved2 : 9;
 		u64 aura : 10;
 		u64 reserved3 : 16;
 	};
 };

 typedef union cvmx_fpa3_load_data cvmx_fpa3_load_data_t;

 /**
  * Struct describing store free operation addresses from FPA pool.
  */
 union cvmx_fpa3_store_addr {
 	u64 u64;
 	struct {
 		u64 seg : 2;
 		u64 reserved1 : 13;
 		u64 io : 1;
 		u64 did : 8;
 		u64 node : 4;
 		u64 reserved2 : 10;
 		u64 aura : 10;
 		u64 fabs : 1;
 		u64 reserved3 : 3;
 		u64 dwb_count : 9;
 		u64 reserved4 : 3;
 	};
 };

 typedef union cvmx_fpa3_store_addr cvmx_fpa3_store_addr_t;

 enum cvmx_fpa3_pool_alignment_e {
 	FPA_NATURAL_ALIGNMENT,
 	FPA_OFFSET_ALIGNMENT,
 	FPA_OPAQUE_ALIGNMENT
 };

 #define CVMX_FPA3_AURAX_LIMIT_MAX ((1ull << 40) - 1)

 /**
  * @INTERNAL
  * Accessor functions to return number of POOLS in an FPA3
  * depending on SoC model.
  * The number is per-node for models supporting multi-node configurations.
  */
 static inline int cvmx_fpa3_num_pools(void)
 {
 	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
 		return 64;
 	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
 		return 32;
 	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
 		return 32;
 	printf("ERROR: %s: Unknowm model\n", __func__);
 	return -1;
 }

 /**
  * @INTERNAL
  * Accessor functions to return number of AURAS in an FPA3
  * depending on SoC model.
  * The number is per-node for models supporting multi-node configurations.
  */
 static inline int cvmx_fpa3_num_auras(void)
 {
 	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
 		return 1024;
 	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
 		return 512;
 	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
 		return 512;
 	printf("ERROR: %s: Unknowm model\n", __func__);
 	return -1;
 }

 /**
  * Get the FPA3 POOL underneath FPA3 AURA, containing all its buffers
  *
  */
 static inline cvmx_fpa3_pool_t cvmx_fpa3_aura_to_pool(cvmx_fpa3_gaura_t aura)
 {
 	cvmx_fpa3_pool_t pool;
 	cvmx_fpa_aurax_pool_t aurax_pool;

 	aurax_pool.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_POOL(aura.laura));

 	pool = __cvmx_fpa3_pool(aura.node, aurax_pool.s.pool);
 	return pool;
 }

 /**
  * Get a new block from the FPA pool
  *
  * @param aura  - aura number
  * Return: pointer to the block or NULL on failure
  */
 static inline void *cvmx_fpa3_alloc(cvmx_fpa3_gaura_t aura)
 {
 	u64 address;
 	cvmx_fpa3_load_data_t load_addr;

 	load_addr.u64 = 0;
 	load_addr.seg = CVMX_MIPS_SPACE_XKPHYS;
 	load_addr.io = 1;
 	load_addr.did = 0x29; /* Device ID. Indicates FPA. */
 	load_addr.node = aura.node;
 	load_addr.red = 0; /* Perform RED on allocation.
 				  * FIXME to use config option
 				  */
 	load_addr.aura = aura.laura;

 	address = cvmx_read64_uint64(load_addr.u64);
 	if (!address)
 		return NULL;
 	return cvmx_phys_to_ptr(address);
 }

 /**
  * Asynchronously get a new block from the FPA
  *
  * The result of cvmx_fpa_async_alloc() may be retrieved using
  * cvmx_fpa_async_alloc_finish().
  *
  * @param scr_addr Local scratch address to put response in.  This is a byte
  *		   address but must be 8 byte aligned.
  * @param aura     Global aura to get the block from
  */
 static inline void cvmx_fpa3_async_alloc(u64 scr_addr, cvmx_fpa3_gaura_t aura)
 {
 	cvmx_fpa3_iobdma_data_t data;

 	/* Hardware only uses 64 bit aligned locations, so convert from byte
 	 * address to 64-bit index
 	 */
 	data.u64 = 0ull;
 	data.cn78xx.scraddr = scr_addr >> 3;
 	data.cn78xx.len = 1;
 	data.cn78xx.did = 0x29;
 	data.cn78xx.node = aura.node;
 	data.cn78xx.aura = aura.laura;
 	cvmx_scratch_write64(scr_addr, 0ull);

 	CVMX_SYNCW;
 	cvmx_send_single(data.u64);
 }

 /**
  * Retrieve the result of cvmx_fpa3_async_alloc
  *
  * @param scr_addr The Local scratch address.  Must be the same value
  * passed to cvmx_fpa_async_alloc().
  *
  * @param aura Global aura the block came from.  Must be the same value
  * passed to cvmx_fpa_async_alloc.
  *
  * Return: Pointer to the block or NULL on failure
  */
 static inline void *cvmx_fpa3_async_alloc_finish(u64 scr_addr, cvmx_fpa3_gaura_t aura)
 {
 	u64 address;

 	CVMX_SYNCIOBDMA;

 	address = cvmx_scratch_read64(scr_addr);
 	if (cvmx_likely(address))
 		return cvmx_phys_to_ptr(address);
 	else
 		/* Try regular alloc if async failed */
 		return cvmx_fpa3_alloc(aura);
 }

 /**
  * Free a pointer back to the pool.
  *
  * @param aura   global aura number
  * @param ptr    physical address of block to free.
  * @param num_cache_lines Cache lines to invalidate
  */
 static inline void cvmx_fpa3_free(void *ptr, cvmx_fpa3_gaura_t aura, unsigned int num_cache_lines)
 {
 	cvmx_fpa3_store_addr_t newptr;
 	cvmx_addr_t newdata;

 	newdata.u64 = cvmx_ptr_to_phys(ptr);

 	/* Make sure that any previous writes to memory go out before we free
 	   this buffer. This also serves as a barrier to prevent GCC from
 	   reordering operations to after the free. */
 	CVMX_SYNCWS;

 	newptr.u64 = 0;
 	newptr.seg = CVMX_MIPS_SPACE_XKPHYS;
 	newptr.io = 1;
 	newptr.did = 0x29; /* Device id, indicates FPA */
 	newptr.node = aura.node;
 	newptr.aura = aura.laura;
 	newptr.fabs = 0; /* Free absolute. FIXME to use config option */
 	newptr.dwb_count = num_cache_lines;

 	cvmx_write_io(newptr.u64, newdata.u64);
 }

 /**
  * Free a pointer back to the pool without flushing the write buffer.
  *
  * @param aura   global aura number
  * @param ptr    physical address of block to free.
  * @param num_cache_lines Cache lines to invalidate
  */
 static inline void cvmx_fpa3_free_nosync(void *ptr, cvmx_fpa3_gaura_t aura,
 					 unsigned int num_cache_lines)
 {
 	cvmx_fpa3_store_addr_t newptr;
 	cvmx_addr_t newdata;

 	newdata.u64 = cvmx_ptr_to_phys(ptr);

 	/* Prevent GCC from reordering writes to (*ptr) */
 	asm volatile("" : : : "memory");

 	newptr.u64 = 0;
 	newptr.seg = CVMX_MIPS_SPACE_XKPHYS;
 	newptr.io = 1;
 	newptr.did = 0x29; /* Device id, indicates FPA */
 	newptr.node = aura.node;
 	newptr.aura = aura.laura;
 	newptr.fabs = 0; /* Free absolute. FIXME to use config option */
 	newptr.dwb_count = num_cache_lines;

 	cvmx_write_io(newptr.u64, newdata.u64);
 }

 static inline int cvmx_fpa3_pool_is_enabled(cvmx_fpa3_pool_t pool)
 {
 	cvmx_fpa_poolx_cfg_t pool_cfg;

 	if (!__cvmx_fpa3_pool_valid(pool))
 		return -1;

 	pool_cfg.u64 = cvmx_read_csr_node(pool.node, CVMX_FPA_POOLX_CFG(pool.lpool));
 	return pool_cfg.cn78xx.ena;
 }

 static inline int cvmx_fpa3_config_red_params(unsigned int node, int qos_avg_en, int red_lvl_dly,
 					      int avg_dly)
 {
 	cvmx_fpa_gen_cfg_t fpa_cfg;
 	cvmx_fpa_red_delay_t red_delay;

 	fpa_cfg.u64 = cvmx_read_csr_node(node, CVMX_FPA_GEN_CFG);
 	fpa_cfg.s.avg_en = qos_avg_en;
 	fpa_cfg.s.lvl_dly = red_lvl_dly;
 	cvmx_write_csr_node(node, CVMX_FPA_GEN_CFG, fpa_cfg.u64);

 	red_delay.u64 = cvmx_read_csr_node(node, CVMX_FPA_RED_DELAY);
 	red_delay.s.avg_dly = avg_dly;
 	cvmx_write_csr_node(node, CVMX_FPA_RED_DELAY, red_delay.u64);
 	return 0;
 }

 /**
  * Gets the buffer size of the specified pool,
  *
  * @param aura Global aura number
  * Return: Returns size of the buffers in the specified pool.
  */
 static inline int cvmx_fpa3_get_aura_buf_size(cvmx_fpa3_gaura_t aura)
 {
 	cvmx_fpa3_pool_t pool;
 	cvmx_fpa_poolx_cfg_t pool_cfg;
 	int block_size;

 	pool = cvmx_fpa3_aura_to_pool(aura);

 	pool_cfg.u64 = cvmx_read_csr_node(pool.node, CVMX_FPA_POOLX_CFG(pool.lpool));
 	block_size = pool_cfg.cn78xx.buf_size << 7;
 	return block_size;
 }

 /**
  * Return the number of available buffers in an AURA
  *
  * @param aura to receive count for
  * Return: available buffer count
  */
 static inline long long cvmx_fpa3_get_available(cvmx_fpa3_gaura_t aura)
 {
 	cvmx_fpa3_pool_t pool;
 	cvmx_fpa_poolx_available_t avail_reg;
 	cvmx_fpa_aurax_cnt_t cnt_reg;
 	cvmx_fpa_aurax_cnt_limit_t limit_reg;
 	long long ret;

 	pool = cvmx_fpa3_aura_to_pool(aura);

 	/* Get POOL available buffer count */
 	avail_reg.u64 = cvmx_read_csr_node(pool.node, CVMX_FPA_POOLX_AVAILABLE(pool.lpool));

 	/* Get AURA current available count */
 	cnt_reg.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT(aura.laura));
 	limit_reg.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LIMIT(aura.laura));

 	if (limit_reg.cn78xx.limit < cnt_reg.cn78xx.cnt)
 		return 0;

 	/* Calculate AURA-based buffer allowance */
 	ret = limit_reg.cn78xx.limit - cnt_reg.cn78xx.cnt;

 	/* Use POOL real buffer availability when less then allowance */
 	if (ret > (long long)avail_reg.cn78xx.count)
 		ret = avail_reg.cn78xx.count;

 	return ret;
 }

 /**
  * Configure the QoS parameters of an FPA3 AURA
  *
  * @param aura is the FPA3 AURA handle
  * @param ena_bp enables backpressure when outstanding count exceeds 'bp_thresh'
  * @param ena_red enables random early discard when outstanding count exceeds 'pass_thresh'
  * @param pass_thresh is the maximum count to invoke flow control
  * @param drop_thresh is the count threshold to begin dropping packets
  * @param bp_thresh is the back-pressure threshold
  *
  */
 static inline void cvmx_fpa3_setup_aura_qos(cvmx_fpa3_gaura_t aura, bool ena_red, u64 pass_thresh,
 					    u64 drop_thresh, bool ena_bp, u64 bp_thresh)
 {
 	unsigned int shift = 0;
 	u64 shift_thresh;
 	cvmx_fpa_aurax_cnt_limit_t limit_reg;
 	cvmx_fpa_aurax_cnt_levels_t aura_level;

 	if (!__cvmx_fpa3_aura_valid(aura))
 		return;

 	/* Get AURAX count limit for validation */
 	limit_reg.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LIMIT(aura.laura));

 	if (pass_thresh < 256)
 		pass_thresh = 255;

 	if (drop_thresh <= pass_thresh || drop_thresh > limit_reg.cn78xx.limit)
 		drop_thresh = limit_reg.cn78xx.limit;

 	if (bp_thresh < 256 || bp_thresh > limit_reg.cn78xx.limit)
 		bp_thresh = limit_reg.cn78xx.limit >> 1;

 	shift_thresh = (bp_thresh > drop_thresh) ? bp_thresh : drop_thresh;

 	/* Calculate shift so that the largest threshold fits in 8 bits */
 	for (shift = 0; shift < (1 << 6); shift++) {
 		if (0 == ((shift_thresh >> shift) & ~0xffull))
 			break;
 	};

 	aura_level.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LEVELS(aura.laura));
 	aura_level.s.pass = pass_thresh >> shift;
 	aura_level.s.drop = drop_thresh >> shift;
 	aura_level.s.bp = bp_thresh >> shift;
 	aura_level.s.shift = shift;
 	aura_level.s.red_ena = ena_red;
 	aura_level.s.bp_ena = ena_bp;
 	cvmx_write_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LEVELS(aura.laura), aura_level.u64);
 }

 cvmx_fpa3_gaura_t cvmx_fpa3_reserve_aura(int node, int desired_aura_num);
 int cvmx_fpa3_release_aura(cvmx_fpa3_gaura_t aura);
 cvmx_fpa3_pool_t cvmx_fpa3_reserve_pool(int node, int desired_pool_num);
 int cvmx_fpa3_release_pool(cvmx_fpa3_pool_t pool);
 int cvmx_fpa3_is_aura_available(int node, int aura_num);
 int cvmx_fpa3_is_pool_available(int node, int pool_num);

 cvmx_fpa3_pool_t cvmx_fpa3_setup_fill_pool(int node, int desired_pool, const char *name,
 					   unsigned int block_size, unsigned int num_blocks,
 					   void *buffer);

 /**
  * Function to attach an aura to an existing pool
  *
  * @param node - configure fpa on this node
  * @param pool - configured pool to attach aura to
  * @param desired_aura - pointer to aura to use, set to -1 to allocate
  * @param name - name to register
  * @param block_size - size of buffers to use
  * @param num_blocks - number of blocks to allocate
  *
  * Return: configured gaura on success, CVMX_FPA3_INVALID_GAURA on failure
  */
 cvmx_fpa3_gaura_t cvmx_fpa3_set_aura_for_pool(cvmx_fpa3_pool_t pool, int desired_aura,
 					      const char *name, unsigned int block_size,
 					      unsigned int num_blocks);

 /**
  * Function to setup and initialize a pool.
  *
  * @param node - configure fpa on this node
  * @param desired_aura - aura to use, -1 for dynamic allocation
  * @param name - name to register
  * @param block_size - size of buffers in pool
  * @param num_blocks - max number of buffers allowed
  */
 cvmx_fpa3_gaura_t cvmx_fpa3_setup_aura_and_pool(int node, int desired_aura, const char *name,
 						void *buffer, unsigned int block_size,
 						unsigned int num_blocks);

 int cvmx_fpa3_shutdown_aura_and_pool(cvmx_fpa3_gaura_t aura);
 int cvmx_fpa3_shutdown_aura(cvmx_fpa3_gaura_t aura);
 int cvmx_fpa3_shutdown_pool(cvmx_fpa3_pool_t pool);
 const char *cvmx_fpa3_get_pool_name(cvmx_fpa3_pool_t pool);
 int cvmx_fpa3_get_pool_buf_size(cvmx_fpa3_pool_t pool);
 const char *cvmx_fpa3_get_aura_name(cvmx_fpa3_gaura_t aura);

 #endif /* __CVMX_FPA3_H__ */
	/* SPDX-License-Identifier: GPL-2.0 */
	/*
	* Copyright (C) 2020 Marvell International Ltd.
	*
	* Interface to the CN78XX Free Pool Allocator, a.k.a. FPA3
	*/

	#include "cvmx-address.h"
	#include "cvmx-fpa-defs.h"
	#include "cvmx-scratch.h"

	#ifndef __CVMX_FPA3_H__
	#define __CVMX_FPA3_H__

	typedef struct {
	unsigned res0 : 6;
	unsigned node : 2;
	unsigned res1 : 2;
	unsigned lpool : 6;
	unsigned valid_magic : 16;
	} cvmx_fpa3_pool_t;

	typedef struct {
	unsigned res0 : 6;
	unsigned node : 2;
	unsigned res1 : 6;
	unsigned laura : 10;
	unsigned valid_magic : 16;
	} cvmx_fpa3_gaura_t;

	#define CVMX_FPA3_VALID_MAGIC 0xf9a3
	#define CVMX_FPA3_INVALID_GAURA ((cvmx_fpa3_gaura_t){ 0, 0, 0, 0, 0 })
	#define CVMX_FPA3_INVALID_POOL ((cvmx_fpa3_pool_t){ 0, 0, 0, 0, 0 })

	static inline bool __cvmx_fpa3_aura_valid(cvmx_fpa3_gaura_t aura)
	{
	if (aura.valid_magic != CVMX_FPA3_VALID_MAGIC)
	return false;
	return true;
	}

	static inline bool __cvmx_fpa3_pool_valid(cvmx_fpa3_pool_t pool)
	{
	if (pool.valid_magic != CVMX_FPA3_VALID_MAGIC)
	return false;
	return true;
	}

	static inline cvmx_fpa3_gaura_t __cvmx_fpa3_gaura(int node, int laura)
	{
	cvmx_fpa3_gaura_t aura;

	if (node < 0)
	node = cvmx_get_node_num();
	if (laura < 0)
	return CVMX_FPA3_INVALID_GAURA;

	aura.node = node;
	aura.laura = laura;
	aura.valid_magic = CVMX_FPA3_VALID_MAGIC;
	return aura;
	}

	static inline cvmx_fpa3_pool_t __cvmx_fpa3_pool(int node, int lpool)
	{
	cvmx_fpa3_pool_t pool;

	if (node < 0)
	node = cvmx_get_node_num();
	if (lpool < 0)
	return CVMX_FPA3_INVALID_POOL;

	pool.node = node;
	pool.lpool = lpool;
	pool.valid_magic = CVMX_FPA3_VALID_MAGIC;
	return pool;
	}

	#undef CVMX_FPA3_VALID_MAGIC

	/**
	* Structure describing the data format used for stores to the FPA.
	*/
	typedef union {
	u64 u64;
	struct {
	u64 scraddr : 8;
	u64 len : 8;
	u64 did : 8;
	u64 addr : 40;
	} s;
	struct {
	u64 scraddr : 8;
	u64 len : 8;
	u64 did : 8;
	u64 node : 4;
	u64 red : 1;
	u64 reserved2 : 9;
	u64 aura : 10;
	u64 reserved3 : 16;
	} cn78xx;
	} cvmx_fpa3_iobdma_data_t;

	/**
	* Struct describing load allocate operation addresses for FPA pool.
	*/
	union cvmx_fpa3_load_data {
	u64 u64;
	struct {
	u64 seg : 2;
	u64 reserved1 : 13;
	u64 io : 1;
	u64 did : 8;
	u64 node : 4;
	u64 red : 1;
	u64 reserved2 : 9;
	u64 aura : 10;
	u64 reserved3 : 16;
	};
	};

	typedef union cvmx_fpa3_load_data cvmx_fpa3_load_data_t;

	/**
	* Struct describing store free operation addresses from FPA pool.
	*/
	union cvmx_fpa3_store_addr {
	u64 u64;
	struct {
	u64 seg : 2;
	u64 reserved1 : 13;
	u64 io : 1;
	u64 did : 8;
	u64 node : 4;
	u64 reserved2 : 10;
	u64 aura : 10;
	u64 fabs : 1;
	u64 reserved3 : 3;
	u64 dwb_count : 9;
	u64 reserved4 : 3;
	};
	};

	typedef union cvmx_fpa3_store_addr cvmx_fpa3_store_addr_t;

	enum cvmx_fpa3_pool_alignment_e {
	FPA_NATURAL_ALIGNMENT,
	FPA_OFFSET_ALIGNMENT,
	FPA_OPAQUE_ALIGNMENT
	};

	#define CVMX_FPA3_AURAX_LIMIT_MAX ((1ull << 40) - 1)

	/**
	* @INTERNAL
	* Accessor functions to return number of POOLS in an FPA3
	* depending on SoC model.
	* The number is per-node for models supporting multi-node configurations.
	*/
	static inline int cvmx_fpa3_num_pools(void)
	{
	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
	return 64;
	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
	return 32;
	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
	return 32;
	printf("ERROR: %s: Unknowm model\n", __func__);
	return -1;
	}

	/**
	* @INTERNAL
	* Accessor functions to return number of AURAS in an FPA3
	* depending on SoC model.
	* The number is per-node for models supporting multi-node configurations.
	*/
	static inline int cvmx_fpa3_num_auras(void)
	{
	if (OCTEON_IS_MODEL(OCTEON_CN78XX))
	return 1024;
	if (OCTEON_IS_MODEL(OCTEON_CNF75XX))
	return 512;
	if (OCTEON_IS_MODEL(OCTEON_CN73XX))
	return 512;
	printf("ERROR: %s: Unknowm model\n", __func__);
	return -1;
	}

	/**
	* Get the FPA3 POOL underneath FPA3 AURA, containing all its buffers
	*
	*/
	static inline cvmx_fpa3_pool_t cvmx_fpa3_aura_to_pool(cvmx_fpa3_gaura_t aura)
	{
	cvmx_fpa3_pool_t pool;
	cvmx_fpa_aurax_pool_t aurax_pool;

	aurax_pool.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_POOL(aura.laura));

	pool = __cvmx_fpa3_pool(aura.node, aurax_pool.s.pool);
	return pool;
	}

	/**
	* Get a new block from the FPA pool
	*
	* @param aura - aura number
	* Return: pointer to the block or NULL on failure
	*/
	static inline void *cvmx_fpa3_alloc(cvmx_fpa3_gaura_t aura)
	{
	u64 address;
	cvmx_fpa3_load_data_t load_addr;

	load_addr.u64 = 0;
	load_addr.seg = CVMX_MIPS_SPACE_XKPHYS;
	load_addr.io = 1;
	load_addr.did = 0x29; /* Device ID. Indicates FPA. */
	load_addr.node = aura.node;
	load_addr.red = 0; /* Perform RED on allocation.
	* FIXME to use config option
	*/
	load_addr.aura = aura.laura;

	address = cvmx_read64_uint64(load_addr.u64);
	if (!address)
	return NULL;
	return cvmx_phys_to_ptr(address);
	}

	/**
	* Asynchronously get a new block from the FPA
	*
	* The result of cvmx_fpa_async_alloc() may be retrieved using
	* cvmx_fpa_async_alloc_finish().
	*
	* @param scr_addr Local scratch address to put response in. This is a byte
	* address but must be 8 byte aligned.
	* @param aura Global aura to get the block from
	*/
	static inline void cvmx_fpa3_async_alloc(u64 scr_addr, cvmx_fpa3_gaura_t aura)
	{
	cvmx_fpa3_iobdma_data_t data;

	/* Hardware only uses 64 bit aligned locations, so convert from byte
	* address to 64-bit index
	*/
	data.u64 = 0ull;
	data.cn78xx.scraddr = scr_addr >> 3;
	data.cn78xx.len = 1;
	data.cn78xx.did = 0x29;
	data.cn78xx.node = aura.node;
	data.cn78xx.aura = aura.laura;
	cvmx_scratch_write64(scr_addr, 0ull);

	CVMX_SYNCW;
	cvmx_send_single(data.u64);
	}

	/**
	* Retrieve the result of cvmx_fpa3_async_alloc
	*
	* @param scr_addr The Local scratch address. Must be the same value
	* passed to cvmx_fpa_async_alloc().
	*
	* @param aura Global aura the block came from. Must be the same value
	* passed to cvmx_fpa_async_alloc.
	*
	* Return: Pointer to the block or NULL on failure
	*/
	static inline void *cvmx_fpa3_async_alloc_finish(u64 scr_addr, cvmx_fpa3_gaura_t aura)
	{
	u64 address;

	CVMX_SYNCIOBDMA;

	address = cvmx_scratch_read64(scr_addr);
	if (cvmx_likely(address))
	return cvmx_phys_to_ptr(address);
	else
	/* Try regular alloc if async failed */
	return cvmx_fpa3_alloc(aura);
	}

	/**
	* Free a pointer back to the pool.
	*
	* @param aura global aura number
	* @param ptr physical address of block to free.
	* @param num_cache_lines Cache lines to invalidate
	*/
	static inline void cvmx_fpa3_free(void *ptr, cvmx_fpa3_gaura_t aura, unsigned int num_cache_lines)
	{
	cvmx_fpa3_store_addr_t newptr;
	cvmx_addr_t newdata;

	newdata.u64 = cvmx_ptr_to_phys(ptr);

	/* Make sure that any previous writes to memory go out before we free
	this buffer. This also serves as a barrier to prevent GCC from
	reordering operations to after the free. */
	CVMX_SYNCWS;

	newptr.u64 = 0;
	newptr.seg = CVMX_MIPS_SPACE_XKPHYS;
	newptr.io = 1;
	newptr.did = 0x29; /* Device id, indicates FPA */
	newptr.node = aura.node;
	newptr.aura = aura.laura;
	newptr.fabs = 0; /* Free absolute. FIXME to use config option */
	newptr.dwb_count = num_cache_lines;

	cvmx_write_io(newptr.u64, newdata.u64);
	}

	/**
	* Free a pointer back to the pool without flushing the write buffer.
	*
	* @param aura global aura number
	* @param ptr physical address of block to free.
	* @param num_cache_lines Cache lines to invalidate
	*/
	static inline void cvmx_fpa3_free_nosync(void *ptr, cvmx_fpa3_gaura_t aura,
	unsigned int num_cache_lines)
	{
	cvmx_fpa3_store_addr_t newptr;
	cvmx_addr_t newdata;

	newdata.u64 = cvmx_ptr_to_phys(ptr);

	/* Prevent GCC from reordering writes to (ptr) /
	asm volatile("" : : : "memory");

	newptr.u64 = 0;
	newptr.seg = CVMX_MIPS_SPACE_XKPHYS;
	newptr.io = 1;
	newptr.did = 0x29; /* Device id, indicates FPA */
	newptr.node = aura.node;
	newptr.aura = aura.laura;
	newptr.fabs = 0; /* Free absolute. FIXME to use config option */
	newptr.dwb_count = num_cache_lines;

	cvmx_write_io(newptr.u64, newdata.u64);
	}

	static inline int cvmx_fpa3_pool_is_enabled(cvmx_fpa3_pool_t pool)
	{
	cvmx_fpa_poolx_cfg_t pool_cfg;

	if (!__cvmx_fpa3_pool_valid(pool))
	return -1;

	pool_cfg.u64 = cvmx_read_csr_node(pool.node, CVMX_FPA_POOLX_CFG(pool.lpool));
	return pool_cfg.cn78xx.ena;
	}

	static inline int cvmx_fpa3_config_red_params(unsigned int node, int qos_avg_en, int red_lvl_dly,
	int avg_dly)
	{
	cvmx_fpa_gen_cfg_t fpa_cfg;
	cvmx_fpa_red_delay_t red_delay;

	fpa_cfg.u64 = cvmx_read_csr_node(node, CVMX_FPA_GEN_CFG);
	fpa_cfg.s.avg_en = qos_avg_en;
	fpa_cfg.s.lvl_dly = red_lvl_dly;
	cvmx_write_csr_node(node, CVMX_FPA_GEN_CFG, fpa_cfg.u64);

	red_delay.u64 = cvmx_read_csr_node(node, CVMX_FPA_RED_DELAY);
	red_delay.s.avg_dly = avg_dly;
	cvmx_write_csr_node(node, CVMX_FPA_RED_DELAY, red_delay.u64);
	return 0;
	}

	/**
	* Gets the buffer size of the specified pool,
	*
	* @param aura Global aura number
	* Return: Returns size of the buffers in the specified pool.
	*/
	static inline int cvmx_fpa3_get_aura_buf_size(cvmx_fpa3_gaura_t aura)
	{
	cvmx_fpa3_pool_t pool;
	cvmx_fpa_poolx_cfg_t pool_cfg;
	int block_size;

	pool = cvmx_fpa3_aura_to_pool(aura);

	pool_cfg.u64 = cvmx_read_csr_node(pool.node, CVMX_FPA_POOLX_CFG(pool.lpool));
	block_size = pool_cfg.cn78xx.buf_size << 7;
	return block_size;
	}

	/**
	* Return the number of available buffers in an AURA
	*
	* @param aura to receive count for
	* Return: available buffer count
	*/
	static inline long long cvmx_fpa3_get_available(cvmx_fpa3_gaura_t aura)
	{
	cvmx_fpa3_pool_t pool;
	cvmx_fpa_poolx_available_t avail_reg;
	cvmx_fpa_aurax_cnt_t cnt_reg;
	cvmx_fpa_aurax_cnt_limit_t limit_reg;
	long long ret;

	pool = cvmx_fpa3_aura_to_pool(aura);

	/* Get POOL available buffer count */
	avail_reg.u64 = cvmx_read_csr_node(pool.node, CVMX_FPA_POOLX_AVAILABLE(pool.lpool));

	/* Get AURA current available count */
	cnt_reg.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT(aura.laura));
	limit_reg.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LIMIT(aura.laura));

	if (limit_reg.cn78xx.limit < cnt_reg.cn78xx.cnt)
	return 0;

	/* Calculate AURA-based buffer allowance */
	ret = limit_reg.cn78xx.limit - cnt_reg.cn78xx.cnt;

	/* Use POOL real buffer availability when less then allowance */
	if (ret > (long long)avail_reg.cn78xx.count)
	ret = avail_reg.cn78xx.count;

	return ret;
	}

	/**
	* Configure the QoS parameters of an FPA3 AURA
	*
	* @param aura is the FPA3 AURA handle
	* @param ena_bp enables backpressure when outstanding count exceeds 'bp_thresh'
	* @param ena_red enables random early discard when outstanding count exceeds 'pass_thresh'
	* @param pass_thresh is the maximum count to invoke flow control
	* @param drop_thresh is the count threshold to begin dropping packets
	* @param bp_thresh is the back-pressure threshold
	*
	*/
	static inline void cvmx_fpa3_setup_aura_qos(cvmx_fpa3_gaura_t aura, bool ena_red, u64 pass_thresh,
	u64 drop_thresh, bool ena_bp, u64 bp_thresh)
	{
	unsigned int shift = 0;
	u64 shift_thresh;
	cvmx_fpa_aurax_cnt_limit_t limit_reg;
	cvmx_fpa_aurax_cnt_levels_t aura_level;

	if (!__cvmx_fpa3_aura_valid(aura))
	return;

	/* Get AURAX count limit for validation */
	limit_reg.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LIMIT(aura.laura));

	if (pass_thresh < 256)
	pass_thresh = 255;

	if (drop_thresh <= pass_thresh \|\| drop_thresh > limit_reg.cn78xx.limit)
	drop_thresh = limit_reg.cn78xx.limit;

	if (bp_thresh < 256 \|\| bp_thresh > limit_reg.cn78xx.limit)
	bp_thresh = limit_reg.cn78xx.limit >> 1;

	shift_thresh = (bp_thresh > drop_thresh) ? bp_thresh : drop_thresh;

	/* Calculate shift so that the largest threshold fits in 8 bits */
	for (shift = 0; shift < (1 << 6); shift++) {
	if (0 == ((shift_thresh >> shift) & ~0xffull))
	break;
	};

	aura_level.u64 = cvmx_read_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LEVELS(aura.laura));
	aura_level.s.pass = pass_thresh >> shift;
	aura_level.s.drop = drop_thresh >> shift;
	aura_level.s.bp = bp_thresh >> shift;
	aura_level.s.shift = shift;
	aura_level.s.red_ena = ena_red;
	aura_level.s.bp_ena = ena_bp;
	cvmx_write_csr_node(aura.node, CVMX_FPA_AURAX_CNT_LEVELS(aura.laura), aura_level.u64);
	}

	cvmx_fpa3_gaura_t cvmx_fpa3_reserve_aura(int node, int desired_aura_num);
	int cvmx_fpa3_release_aura(cvmx_fpa3_gaura_t aura);
	cvmx_fpa3_pool_t cvmx_fpa3_reserve_pool(int node, int desired_pool_num);
	int cvmx_fpa3_release_pool(cvmx_fpa3_pool_t pool);
	int cvmx_fpa3_is_aura_available(int node, int aura_num);
	int cvmx_fpa3_is_pool_available(int node, int pool_num);

	cvmx_fpa3_pool_t cvmx_fpa3_setup_fill_pool(int node, int desired_pool, const char *name,
	unsigned int block_size, unsigned int num_blocks,
	void *buffer);

	/**
	* Function to attach an aura to an existing pool
	*
	* @param node - configure fpa on this node
	* @param pool - configured pool to attach aura to
	* @param desired_aura - pointer to aura to use, set to -1 to allocate
	* @param name - name to register
	* @param block_size - size of buffers to use
	* @param num_blocks - number of blocks to allocate
	*
	* Return: configured gaura on success, CVMX_FPA3_INVALID_GAURA on failure
	*/
	cvmx_fpa3_gaura_t cvmx_fpa3_set_aura_for_pool(cvmx_fpa3_pool_t pool, int desired_aura,
	const char *name, unsigned int block_size,
	unsigned int num_blocks);

	/**
	* Function to setup and initialize a pool.
	*
	* @param node - configure fpa on this node
	* @param desired_aura - aura to use, -1 for dynamic allocation
	* @param name - name to register
	* @param block_size - size of buffers in pool
	* @param num_blocks - max number of buffers allowed
	*/
	cvmx_fpa3_gaura_t cvmx_fpa3_setup_aura_and_pool(int node, int desired_aura, const char *name,
	void *buffer, unsigned int block_size,
	unsigned int num_blocks);

	int cvmx_fpa3_shutdown_aura_and_pool(cvmx_fpa3_gaura_t aura);
	int cvmx_fpa3_shutdown_aura(cvmx_fpa3_gaura_t aura);
	int cvmx_fpa3_shutdown_pool(cvmx_fpa3_pool_t pool);
	const char *cvmx_fpa3_get_pool_name(cvmx_fpa3_pool_t pool);
	int cvmx_fpa3_get_pool_buf_size(cvmx_fpa3_pool_t pool);
	const char *cvmx_fpa3_get_aura_name(cvmx_fpa3_gaura_t aura);

	#endif /* __CVMX_FPA3_H__ */