GNUnet  0.10.x
Data Structures | Macros | Enumerations | Functions
plugin_ats_ril.c File Reference

ATS reinforcement learning solver. More...

#include "platform.h"
#include <float.h>
#include <math.h>
#include "gnunet_ats_plugin.h"
#include "gnunet-service-ats_addresses.h"
Include dependency graph for plugin_ats_ril.c:

Go to the source code of this file.

Data Structures

struct  RIL_Learning_Parameters
 Global learning parameters. More...
 
struct  RIL_Address_Wrapped
 Wrapper for addresses to store them in agent's linked list. More...
 
struct  RIL_Peer_Agent
 
struct  RIL_Scope
 
struct  GAS_RIL_Handle
 A handle for the reinforcement learning solver. More...
 

Macros

#define LOG(kind, ...)   GNUNET_log_from (kind, "ats-ril",__VA_ARGS__)
 
#define RIL_MIN_BW   (5 * ntohl (GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__))
 
#define RIL_MAX_BW   GNUNET_ATS_MaxBandwidth
 
#define RIL_ACTION_INVALID   -1
 
#define RIL_INTERVAL_EXPONENT   10
 
#define RIL_UTILITY_DELAY_MAX   1000
 
#define RIL_DEFAULT_STEP_TIME_MIN   GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 200)
 
#define RIL_DEFAULT_STEP_TIME_MAX   GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 2000)
 
#define RIL_DEFAULT_ALGORITHM   RIL_ALGO_SARSA
 
#define RIL_DEFAULT_SELECT   RIL_SELECT_SOFTMAX
 
#define RIL_DEFAULT_WELFARE   RIL_WELFARE_NASH
 
#define RIL_DEFAULT_DISCOUNT_BETA   0.6
 
#define RIL_DEFAULT_DISCOUNT_GAMMA   0.5
 
#define RIL_DEFAULT_GRADIENT_STEP_SIZE   0.01
 
#define RIL_DEFAULT_TRACE_DECAY   0.5
 
#define RIL_DEFAULT_EXPLORE_RATIO   1
 
#define RIL_DEFAULT_EXPLORE_DECAY   0.95
 
#define RIL_DEFAULT_RBF_DIVISOR   50
 
#define RIL_DEFAULT_TEMPERATURE   0.1
 
#define RIL_DEFAULT_TEMPERATURE_DECAY   1
 
#define RIL_INC_DEC_STEP_SIZE   1
 
#define RIL_NOP_DECAY   0.5
 

Enumerations

enum  RIL_Action_Type {
  RIL_ACTION_NOTHING = 0, RIL_ACTION_BW_IN_DBL = -2, RIL_ACTION_BW_IN_HLV = -3, RIL_ACTION_BW_IN_INC = 1,
  RIL_ACTION_BW_IN_DEC = 2, RIL_ACTION_BW_OUT_DBL = -4, RIL_ACTION_BW_OUT_HLV = -5, RIL_ACTION_BW_OUT_INC = 3,
  RIL_ACTION_BW_OUT_DEC = 4, RIL_ACTION_TYPE_NUM = 5
}
 ATS reinforcement learning solver. More...
 
enum  RIL_Algorithm { RIL_ALGO_SARSA = 0, RIL_ALGO_Q = 1 }
 
enum  RIL_Select { RIL_SELECT_SOFTMAX = 0, RIL_SELECT_EGREEDY = 1 }
 
enum  RIL_Welfare { RIL_WELFARE_NASH, RIL_WELFARE_EGALITARIAN }
 
enum  RIL_E_Modification { RIL_E_DECAY, RIL_E_ZERO, RIL_E_ACCUMULATE, RIL_E_REPLACE }
 

Functions

static double agent_q (struct RIL_Peer_Agent *agent, const double *state, int action)
 Estimate the current action-value for state s and action a. More...
 
static int agent_address_get_index (struct RIL_Peer_Agent *agent, struct ATS_Address *address)
 Get the index of the address in the agent's list. More...
 
static struct RIL_Address_Wrappedagent_address_get_wrapped (struct RIL_Peer_Agent *agent, struct ATS_Address *address)
 Gets the wrapped address from the agent's list. More...
 
static int agent_action_is_possible (struct RIL_Peer_Agent *agent, int action)
 
static int agent_get_action_max (struct RIL_Peer_Agent *agent, double *state)
 Gets the action, with the maximal estimated Q-value (i.e. More...
 
static int agent_get_action_random (struct RIL_Peer_Agent *agent)
 Chooses a random action from the set of possible ones. More...
 
static void agent_update (struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
 Updates the weights (i.e. More...
 
static void agent_modify_eligibility (struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
 Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each component as in accumulating eligibility traces RIL_E_REPLACE - resets each component to feature as in replacing traces RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing. More...
 
static void ril_inform (struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
 Informs the environment about the status of the solver. More...
 
static unsigned long long ril_get_max_bw (struct RIL_Scope *net)
 Calculates the maximum bandwidth an agent can assign in a network scope. More...
 
static void envi_set_active_suggestion (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
 Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify ATS of its new decision. More...
 
static double * envi_get_state (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Allocates a state vector and fills it with the features present. More...
 
static double agent_get_utility (struct RIL_Peer_Agent *agent)
 Returns the utility value of the connection an agent manages. More...
 
static double ril_network_get_social_welfare (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
 Calculates the social welfare within a network scope according to what social welfare measure is set in the configuration. More...
 
static double envi_get_penalty (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 
static double envi_get_reward (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Gets the reward for the last performed step, which is calculated in equal parts from the local (the peer specific) and the global (for all peers identical) reward. More...
 
static void envi_action_bw_double (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Doubles the bandwidth for the active address. More...
 
static void envi_action_bw_halven (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Cuts the bandwidth for the active address in half. More...
 
static void envi_action_bw_inc (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Increases the bandwidth by 5 times the minimum bandwidth for the active address. More...
 
static void envi_action_bw_dec (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Decreases the bandwidth by 5 times the minimum bandwidth for the active address. More...
 
static void envi_action_address_switch (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
 Switches to the address given by its index. More...
 
static void envi_do_action (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
 Puts the action into effect by calling the according function. More...
 
static int agent_select_egreedy (struct RIL_Peer_Agent *agent, double *state)
 Selects the next action using the e-greedy strategy. More...
 
static int agent_select_softmax (struct RIL_Peer_Agent *agent, double *state)
 Selects the next action with a probability corresponding to its value. More...
 
static int agent_select_action (struct RIL_Peer_Agent *agent, double *state)
 Select the next action of an agent either according to the e-greedy strategy or the softmax strategy. More...
 
static void agent_step (struct RIL_Peer_Agent *agent)
 Performs one step of the Markov Decision Process. More...
 
static void ril_step (struct GAS_RIL_Handle *solver)
 Prototype of the ril_step() procedure. More...
 
static void ril_step_scheduler_task (void *cls)
 Task for the scheduler, which performs one step and lets the solver know that no further step is scheduled. More...
 
static double ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver)
 Determines how much of the available bandwidth is assigned. More...
 
static struct RIL_Scoperil_get_network (struct GAS_RIL_Handle *s, uint32_t type)
 Lookup network struct by type. More...
 
static int ril_network_is_not_full (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
 Determines whether more connections are allocated in a network scope, than they would theoretically fit. More...
 
static void ril_try_unblock_agent (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
 Unblocks an agent for which a connection request is there, that could not be satisfied. More...
 
static void ril_calculate_discount (struct GAS_RIL_Handle *solver)
 Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step. More...
 
static int ril_network_count_active_agents (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
 Count the number of active agents/connections in a network scope. More...
 
static unsigned long long ril_network_get_assigned (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
 Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the outbound direction. More...
 
static unsigned long long ril_network_get_utilized (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
 Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound or in the outbound direction. More...
 
static void ril_networks_update_state (struct GAS_RIL_Handle *solver)
 Retrieves the state of the network scope, so that its attributes are up-to- date. More...
 
static void ril_step_schedule_next (struct GAS_RIL_Handle *solver)
 Schedules the next global step in an adaptive way. More...
 
static void agent_w_init (struct RIL_Peer_Agent *agent)
 Initializes the matrix W of parameter vectors theta with small random numbers. More...
 
static struct RIL_Peer_Agentagent_init (void *s, const struct GNUNET_PeerIdentity *peer)
 Initialize an agent without addresses and its knowledge base. More...
 
static void agent_die (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Deallocate agent. More...
 
static struct RIL_Peer_Agentril_get_agent (struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
 Returns the agent for a peer. More...
 
static int ril_network_is_active (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
 Determine whether at least the minimum bandwidth is set for the network. More...
 
static void ril_cut_from_vector (void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
 Cuts a slice out of a vector of elements. More...
 
static void GAS_ril_address_change_preference (void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
 Change relative preference for quality in solver. More...
 
static void GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network)
 Add a new address for a peer to the solver. More...
 
static void GAS_ril_address_delete (void *solver, struct ATS_Address *address)
 Delete an address in the solver. More...
 
static void GAS_ril_address_property_changed (void *solver, struct ATS_Address *address)
 Update the properties of an address in the solver. More...
 
static void GAS_ril_address_preference_feedback (void *solver, struct GNUNET_SERVICE_Client *application, const struct GNUNET_PeerIdentity *peer, const struct GNUNET_TIME_Relative scope, enum GNUNET_ATS_PreferenceKind kind, double score)
 Give feedback about the current assignment. More...
 
static void GAS_ril_bulk_start (void *solver)
 Start a bulk operation. More...
 
static void GAS_ril_bulk_stop (void *solver)
 Bulk operation done. More...
 
static void GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer)
 Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth changed callback. More...
 
static void GAS_ril_stop_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer)
 Tell solver stop notifying ATS about changes for this peers. More...
 
void * libgnunet_plugin_ats_ril_init (void *cls)
 Entry point for the plugin. More...
 
void * libgnunet_plugin_ats_ril_done (void *cls)
 Exit point for the plugin. More...
 

Detailed Description

ATS reinforcement learning solver.

Author
Fabian Oehlmann
Matthias Wachs

Definition in file plugin_ats_ril.c.

Macro Definition Documentation

◆ LOG

#define LOG (   kind,
  ... 
)    GNUNET_log_from (kind, "ats-ril",__VA_ARGS__)

◆ RIL_MIN_BW

#define RIL_MIN_BW   (5 * ntohl (GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__))

◆ RIL_MAX_BW

#define RIL_MAX_BW   GNUNET_ATS_MaxBandwidth

Definition at line 38 of file plugin_ats_ril.c.

Referenced by agent_action_is_possible().

◆ RIL_ACTION_INVALID

#define RIL_ACTION_INVALID   -1

◆ RIL_INTERVAL_EXPONENT

#define RIL_INTERVAL_EXPONENT   10

Definition at line 41 of file plugin_ats_ril.c.

Referenced by ril_step_schedule_next().

◆ RIL_UTILITY_DELAY_MAX

#define RIL_UTILITY_DELAY_MAX   1000

Definition at line 42 of file plugin_ats_ril.c.

Referenced by agent_get_utility().

◆ RIL_DEFAULT_STEP_TIME_MIN

#define RIL_DEFAULT_STEP_TIME_MIN   GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 200)

Definition at line 44 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_STEP_TIME_MAX

#define RIL_DEFAULT_STEP_TIME_MAX   GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 2000)

Definition at line 45 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_ALGORITHM

#define RIL_DEFAULT_ALGORITHM   RIL_ALGO_SARSA

Definition at line 46 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_SELECT

#define RIL_DEFAULT_SELECT   RIL_SELECT_SOFTMAX

Definition at line 47 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_WELFARE

#define RIL_DEFAULT_WELFARE   RIL_WELFARE_NASH

Definition at line 48 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_DISCOUNT_BETA

#define RIL_DEFAULT_DISCOUNT_BETA   0.6

Definition at line 49 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_DISCOUNT_GAMMA

#define RIL_DEFAULT_DISCOUNT_GAMMA   0.5

Definition at line 50 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_GRADIENT_STEP_SIZE

#define RIL_DEFAULT_GRADIENT_STEP_SIZE   0.01

Definition at line 51 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TRACE_DECAY

#define RIL_DEFAULT_TRACE_DECAY   0.5

Definition at line 52 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_EXPLORE_RATIO

#define RIL_DEFAULT_EXPLORE_RATIO   1

Definition at line 53 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_EXPLORE_DECAY

#define RIL_DEFAULT_EXPLORE_DECAY   0.95

Definition at line 54 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_RBF_DIVISOR

#define RIL_DEFAULT_RBF_DIVISOR   50

Definition at line 55 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TEMPERATURE

#define RIL_DEFAULT_TEMPERATURE   0.1

Definition at line 56 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TEMPERATURE_DECAY

#define RIL_DEFAULT_TEMPERATURE_DECAY   1

Definition at line 57 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_INC_DEC_STEP_SIZE

#define RIL_INC_DEC_STEP_SIZE   1

Definition at line 59 of file plugin_ats_ril.c.

Referenced by envi_action_bw_dec(), and envi_action_bw_inc().

◆ RIL_NOP_DECAY

#define RIL_NOP_DECAY   0.5

Definition at line 60 of file plugin_ats_ril.c.

Referenced by envi_get_reward().

Enumeration Type Documentation

◆ RIL_Action_Type

ATS reinforcement learning solver.

General description The actions, how an agent can manipulate the current assignment. I.e. how the bandwidth can be changed for the currently chosen address. Not depicted in the enum are the actions of switching to a particular address. The action of switching to address with index i is depicted by the number (RIL_ACTION_TYPE_NUM + i).

Enumerator
RIL_ACTION_NOTHING 
RIL_ACTION_BW_IN_DBL 
RIL_ACTION_BW_IN_HLV 
RIL_ACTION_BW_IN_INC 
RIL_ACTION_BW_IN_DEC 
RIL_ACTION_BW_OUT_DBL 
RIL_ACTION_BW_OUT_HLV 
RIL_ACTION_BW_OUT_INC 
RIL_ACTION_BW_OUT_DEC 
RIL_ACTION_TYPE_NUM 

Definition at line 74 of file plugin_ats_ril.c.

◆ RIL_Algorithm

Enumerator
RIL_ALGO_SARSA 
RIL_ALGO_Q 

Definition at line 88 of file plugin_ats_ril.c.

89 {
90  RIL_ALGO_SARSA = 0,
91  RIL_ALGO_Q = 1
92 };

◆ RIL_Select

enum RIL_Select
Enumerator
RIL_SELECT_SOFTMAX 
RIL_SELECT_EGREEDY 

Definition at line 94 of file plugin_ats_ril.c.

◆ RIL_Welfare

Enumerator
RIL_WELFARE_NASH 
RIL_WELFARE_EGALITARIAN 

Definition at line 100 of file plugin_ats_ril.c.

◆ RIL_E_Modification

Enumerator
RIL_E_DECAY 
RIL_E_ZERO 
RIL_E_ACCUMULATE 
RIL_E_REPLACE 

Definition at line 106 of file plugin_ats_ril.c.

Function Documentation

◆ agent_q()

static double agent_q ( struct RIL_Peer_Agent agent,
const double *  state,
int  action 
)
static

Estimate the current action-value for state s and action a.

Parameters
agentagent performing the estimation
states
actiona
Returns
estimation value

Definition at line 479 of file plugin_ats_ril.c.

References RIL_Peer_Agent::m, result, and RIL_Peer_Agent::W.

Referenced by agent_get_action_max(), agent_select_softmax(), and agent_update().

482 {
483  unsigned int i;
484  double result = 0.0;
485 
486  for (i = 0; i < agent->m; i++)
487  result += state[i] * agent->W[action][i];
488 
489  /* prevent crashes if learning diverges */
490  if (isnan(result))
491  return isnan(result) * UINT32_MAX;
492  if (isinf(result))
493  return isinf(result) * UINT32_MAX;
494  return result;
495 }
unsigned int m
Number of rows of W / Number of state-vector features.
enum State state
current state of profiling
static int result
Global testing status.
double ** W
Experience matrix W.
Here is the caller graph for this function:

◆ agent_address_get_index()

static int agent_address_get_index ( struct RIL_Peer_Agent agent,
struct ATS_Address address 
)
static

Get the index of the address in the agent's list.

Parameters
agentagent handle
addressaddress handle
Returns
the index, starting with zero

Definition at line 506 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, and RIL_Address_Wrapped::next.

Referenced by agent_action_is_possible(), envi_do_action(), envi_get_state(), and GAS_ril_address_delete().

507 {
508  int i;
509  struct RIL_Address_Wrapped *cur;
510 
511  i = -1;
512  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
513  {
514  i++;
515  if (cur->address_naked == address)
516  return i;
517  }
518  return i;
519 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
struct ATS_Address * address_naked
The address.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the caller graph for this function:

◆ agent_address_get_wrapped()

static struct RIL_Address_Wrapped* agent_address_get_wrapped ( struct RIL_Peer_Agent agent,
struct ATS_Address address 
)
static

Gets the wrapped address from the agent's list.

Parameters
agentagent handle
addressaddress handle
Returns
wrapped address

Definition at line 530 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, and RIL_Address_Wrapped::next.

Referenced by GAS_ril_address_delete().

531 {
532  struct RIL_Address_Wrapped *cur;
533 
534  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
535  if (cur->address_naked == address)
536  return cur;
537  return NULL;
538 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
struct ATS_Address * address_naked
The address.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the caller graph for this function:

◆ agent_action_is_possible()

static int agent_action_is_possible ( struct RIL_Peer_Agent agent,
int  action 
)
static

Definition at line 542 of file plugin_ats_ril.c.

References ATS_Address::active, RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_assert, GNUNET_NO, GNUNET_YES, RIL_ACTION_BW_IN_DBL, RIL_ACTION_BW_IN_DEC, RIL_ACTION_BW_IN_HLV, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_DBL, RIL_ACTION_BW_OUT_DEC, RIL_ACTION_BW_OUT_HLV, RIL_ACTION_BW_OUT_INC, RIL_ACTION_NOTHING, RIL_ACTION_TYPE_NUM, and RIL_MAX_BW.

Referenced by agent_get_action_max(), agent_get_action_random(), and agent_select_softmax().

543 {
544  int address_index;
545 
546  switch (action)
547  {
548  case RIL_ACTION_NOTHING:
549  return GNUNET_YES;
550  break;
553  if (agent->bw_in >= RIL_MAX_BW)
554  return GNUNET_NO;
555  else
556  return GNUNET_YES;
557  break;
560  if (agent->bw_in <= 0)
561  return GNUNET_NO;
562  else
563  return GNUNET_YES;
564  break;
567  if (agent->bw_out >= RIL_MAX_BW)
568  return GNUNET_NO;
569  else
570  return GNUNET_YES;
571  break;
574  if (agent->bw_out <= 0)
575  return GNUNET_NO;
576  else
577  return GNUNET_YES;
578  break;
579  default:
580  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) //switch address action
581  {
582  address_index = action - RIL_ACTION_TYPE_NUM;
583 
584  GNUNET_assert(address_index >= 0);
586  address_index <= agent_address_get_index (agent, agent->addresses_tail->address_naked));
587 
588  if ((agent_address_get_index(agent, agent->address_inuse) == address_index) ||
589  agent->address_inuse->active)
590  return GNUNET_NO;
591  else
592  return GNUNET_YES;
593  break;
594  }
595  // error - action does not exist
597  }
598 }
#define RIL_MAX_BW
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int active
Is this the active address for this peer?
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:81
struct ATS_Address * address_naked
The address.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define GNUNET_YES
Definition: gnunet_common.h:80
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_action_max()

static int agent_get_action_max ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Gets the action, with the maximal estimated Q-value (i.e.

the one currently estimated to bring the most reward in the future)

Parameters
agentagent performing the calculation
statethe state from which to take the action
Returns
the action promising most future reward

Definition at line 610 of file plugin_ats_ril.c.

References agent_action_is_possible(), agent_q(), GNUNET_assert, RIL_Peer_Agent::n, and RIL_ACTION_INVALID.

Referenced by agent_select_egreedy(), agent_select_softmax(), and agent_step().

611 {
612  int i;
613  int max_i = RIL_ACTION_INVALID;
614  double cur_q;
615  double max_q = -DBL_MAX;
616 
617  for (i = 0; i < agent->n; i++)
618  {
619  if (agent_action_is_possible(agent, i))
620  {
621  cur_q = agent_q (agent, state, i);
622  if (cur_q > max_q)
623  {
624  max_q = cur_q;
625  max_i = i;
626  }
627  }
628  }
629 
631 
632  return max_i;
633 }
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
enum State state
current state of profiling
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_action_random()

static int agent_get_action_random ( struct RIL_Peer_Agent agent)
static

Chooses a random action from the set of possible ones.

Parameters
agentthe agent performing the action
Returns
the action index

Definition at line 642 of file plugin_ats_ril.c.

References agent_action_is_possible(), GNUNET_assert, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::n, and RIL_ACTION_INVALID.

Referenced by agent_select_egreedy().

643 {
644  int i;
645  int is_possible[agent->n];
646  int sum = 0;
647  int r;
648 
649  for (i = 0; i<agent->n; i++)
650  {
651  if (agent_action_is_possible(agent, i))
652  {
653  is_possible[i] = GNUNET_YES;
654  sum++;
655  }
656  else
657  {
658  is_possible[i] = GNUNET_NO;
659  }
660  }
661 
663 
664  sum = -1;
665  for (i = 0; i<agent->n; i++)
666  {
667  if (is_possible[i])
668  {
669  sum++;
670  if (sum == r)
671  return i;
672  }
673  }
674 
676  return RIL_ACTION_INVALID;
677 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:81
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_YES
Definition: gnunet_common.h:80
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_update()

static void agent_update ( struct RIL_Peer_Agent agent,
double  reward,
double *  s_next,
int  a_prime 
)
static

Updates the weights (i.e.

coefficients) of the weight vector in matrix W for action a

Parameters
agentthe agent performing the update
rewardthe reward received for the last action
s_nextthe new state, the last step got the agent into
a_primethe new

Definition at line 689 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, agent_q(), RIL_Learning_Parameters::alpha, delta, RIL_Peer_Agent::E, RIL_Peer_Agent::envi, GAS_RIL_Handle::global_discount_integrated, GAS_RIL_Handle::global_discount_variable, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, RIL_Peer_Agent::s_old, and RIL_Peer_Agent::W.

Referenced by agent_step().

690 {
691  int i;
692  int k;
693  double delta;
694  double **theta = agent->W;
695 
696  delta = agent->envi->global_discount_integrated * reward; //reward
697  delta += agent->envi->global_discount_variable * agent_q (agent, s_next, a_prime); //discounted future value
698  delta -= agent_q (agent, agent->s_old, agent->a_old); //one step
699 
700 // LOG(GNUNET_ERROR_TYPE_INFO, "update() Step# %llu Q(s,a): %f a: %f r: %f y: %f Q(s+1,a+1) = %f delta: %f\n",
701 // agent->step_count,
702 // agent_q (agent, agent->s_old, agent->a_old),
703 // agent->envi->parameters.alpha,
704 // reward,
705 // agent->envi->global_discount_variable,
706 // agent_q (agent, s_next, a_prime),
707 // delta);
708 
709  for (k = 0; k < agent->n; k++)
710  {
711  for (i = 0; i < agent->m; i++)
712  {
713  // LOG(GNUNET_ERROR_TYPE_INFO, "alpha = %f delta = %f e[%d] = %f\n",
714  // agent->envi->parameters.alpha,
715  // delta,
716  // i,
717  // agent->e[i]);
718  theta[k][i] += agent->envi->parameters.alpha * delta * agent->E[k][i];
719  }
720  }
721 }
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
unsigned int m
Number of rows of W / Number of state-vector features.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
int a_old
Last chosen action.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
double ** E
Eligibility traces.
double global_discount_variable
Variable discount factor, dependent on time between steps.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
struct GAS_RIL_Handle * envi
Environment handle.
double alpha
Gradient-descent step-size.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_modify_eligibility()

static void agent_modify_eligibility ( struct RIL_Peer_Agent agent,
enum RIL_E_Modification  mod,
double *  feature,
int  action 
)
static

Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each component as in accumulating eligibility traces RIL_E_REPLACE - resets each component to feature as in replacing traces RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing.

Parameters
agentthe agent handle
modthe kind of modification
featurethe feature vector
actionthe action to take

Definition at line 737 of file plugin_ats_ril.c.

References RIL_Peer_Agent::E, RIL_Peer_Agent::envi, GAS_RIL_Handle::global_discount_variable, RIL_Learning_Parameters::lambda, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, RIL_E_ACCUMULATE, RIL_E_DECAY, RIL_E_REPLACE, and RIL_E_ZERO.

Referenced by agent_step().

741 {
742  int i;
743  int k;
744 
745  for (i = 0; i < agent->m; i++)
746  {
747  switch (mod)
748  {
749  case RIL_E_ACCUMULATE:
750  agent->E[action][i] += feature[i];
751  break;
752  case RIL_E_REPLACE:
753  agent->E[action][i] = agent->E[action][i] > feature[i] ? agent->E[action][i] : feature[i];
754  break;
755  case RIL_E_DECAY:
756  for (k = 0; k < agent->n; k++)
757  {
758  agent->E[k][i] *= agent->envi->global_discount_variable * agent->envi->parameters.lambda;
759  }
760  break;
761  case RIL_E_ZERO:
762  for (k = 0; k < agent->n; k++)
763  {
764  agent->E[k][i] = 0;
765  }
766  break;
767  }
768  }
769 }
double lambda
Trace-decay factor for eligibility traces.
unsigned int m
Number of rows of W / Number of state-vector features.
struct RIL_Learning_Parameters parameters
Learning parameters.
double ** E
Eligibility traces.
double global_discount_variable
Variable discount factor, dependent on time between steps.
unsigned int n
Number of columns of W / Number of actions.
struct GAS_RIL_Handle * envi
Environment handle.
Here is the caller graph for this function:

◆ ril_inform()

static void ril_inform ( struct GAS_RIL_Handle solver,
enum GAS_Solver_Operation  op,
enum GAS_Solver_Status  stat 
)
static

Informs the environment about the status of the solver.

Parameters
solver
op
stat

Definition at line 779 of file plugin_ats_ril.c.

References GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, GAS_INFO_NONE, and GNUNET_ATS_PluginEnvironment::info_cb.

Referenced by ril_step().

782 {
783  solver->env->info_cb (solver->env->cls,
784  op,
785  stat,
786  GAS_INFO_NONE);
787 }
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
No more specific information.
GAS_solver_information_callback info_cb
Callback for solver to call with status information, can be NULL.
void * cls
Closure to pass to all callbacks in this struct.
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:139
Here is the caller graph for this function:

◆ ril_get_max_bw()

static unsigned long long ril_get_max_bw ( struct RIL_Scope net)
static

Calculates the maximum bandwidth an agent can assign in a network scope.

Parameters
net

Definition at line 795 of file plugin_ats_ril.c.

References RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, GNUNET_ATS_MaxBandwidth, GNUNET_MAX, and GNUNET_MIN.

Referenced by envi_action_bw_double(), envi_action_bw_inc(), and envi_get_state().

796 {
798 }
#define GNUNET_MAX(a, b)
Definition: gnunet_common.h:85
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:83
#define GNUNET_ATS_MaxBandwidth
Maximum bandwidth assigned to a network : 4095 MB/s.
uint32_t bw_out_available
Total available outbound bandwidth.
uint32_t bw_in_available
Total available inbound bandwidth.
Here is the caller graph for this function:

◆ envi_set_active_suggestion()

static void envi_set_active_suggestion ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
struct ATS_Address new_address,
unsigned long long  new_bw_in,
unsigned long long  new_bw_out,
int  silent 
)
static

Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify ATS of its new decision.

Parameters
solversolver handle
agentagent handle
new_addressthe address which is to be used
new_bw_inthe new amount of inbound bandwidth set for this address
new_bw_outthe new amount of outbound bandwidth set for this address
silentdisables invocation of the bw_changed callback, if GNUNET_YES

Definition at line 812 of file plugin_ats_ril.c.

References ATS_Address::active, RIL_Peer_Agent::address_inuse, ATS_Address::assigned_bw_in, ATS_Address::assigned_bw_out, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::is_active, LOG, notify, RIL_Peer_Agent::peer, RIL_Peer_Agent::suggestion_address, and RIL_Peer_Agent::suggestion_issue.

Referenced by envi_action_address_switch(), envi_action_bw_dec(), envi_action_bw_double(), envi_action_bw_halven(), envi_action_bw_inc(), GAS_ril_address_delete(), GAS_ril_get_preferred_address(), GAS_ril_stop_get_preferred_address(), and ril_try_unblock_agent().

818 {
819  int notify = GNUNET_NO;
820 
822  " set_active_suggestion() for peer '%s'\n",
823  GNUNET_i2s (&agent->peer));
824 
825  //address change
826  if (agent->address_inuse != new_address)
827  {
828  if (NULL != agent->address_inuse)
829  {
830  agent->address_inuse->active = GNUNET_NO;
831  agent->address_inuse->assigned_bw_in = 0;
832  agent->address_inuse->assigned_bw_out = 0;
833  }
834  if (NULL != new_address)
835  {
836  LOG(GNUNET_ERROR_TYPE_DEBUG, " set address active: %s\n", agent->is_active ? "yes" : "no");
837  new_address->active = agent->is_active;
838  new_address->assigned_bw_in = agent->bw_in;
839  new_address->assigned_bw_out = agent->bw_out;
840  }
841  notify |= GNUNET_YES;
842  }
843 
844  if (new_address)
845  {
846  //activity change
847  if (new_address->active != agent->is_active)
848  {
849  new_address->active = agent->is_active;
850  notify |= GNUNET_YES;
851  }
852 
853  //bw change
854  if (agent->bw_in != new_bw_in)
855  {
856  agent->bw_in = new_bw_in;
857  new_address->assigned_bw_in = new_bw_in;
858  notify |= GNUNET_YES;
859  }
860  if (agent->bw_out != new_bw_out)
861  {
862  agent->bw_out = new_bw_out;
863  new_address->assigned_bw_out = new_bw_out;
864  notify |= GNUNET_YES;
865  }
866  }
867 
868  if (notify && agent->is_active && (GNUNET_NO == silent))
869  {
870  if (new_address)
871  {
872  LOG(GNUNET_ERROR_TYPE_DEBUG, " envi_set_active_suggestion() notify\n");
873  agent->suggestion_issue = GNUNET_YES;
874  agent->suggestion_address = new_address;
875  }
876  else if (agent->address_inuse)
877  {
878  /* disconnect case, no new address */
881  agent->bw_in = 0;
882  agent->bw_out = 0;
883 
884  agent->suggestion_issue = GNUNET_YES;
885  agent->suggestion_address = agent->address_inuse;
886  }
887  }
888  agent->address_inuse = new_address;
889 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
int active
Is this the active address for this peer?
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:81
int suggestion_issue
Flag whether a suggestion has to be issued.
uint32_t assigned_bw_in
Inbound bandwidth assigned by solver.
struct GNUNET_PeerIdentity peer
Peer ID.
static int silent
–silent option
Definition: gnunet-qr.c:49
static struct GNUNET_PEERINFO_NotifyContext * notify
Handle to the peerinfo notify service (NULL until we&#39;ve connected to it).
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
uint32_t assigned_bw_out
Outbound bandwidth assigned by solver.
#define GNUNET_YES
Definition: gnunet_common.h:80
#define LOG(kind,...)
struct ATS_Address * suggestion_address
The address which has to be issued.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_get_state()

static double* envi_get_state ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Allocates a state vector and fills it with the features present.

Parameters
solverthe solver handle
agentthe agent handle
Returns
pointer to the state vector

Definition at line 899 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, agent_address_get_index(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, removetrailingwhitespace::f, GNUNET_malloc, m, RIL_Peer_Agent::m, GAS_RIL_Handle::parameters, RIL_Learning_Parameters::rbf_divisor, ril_get_max_bw(), ATS_Address::solver_information, and state.

Referenced by agent_step().

900 {
901  double *state;
902  double y[2];
903  double x[2];
904  double d[2];
905  double sigma;
906  double f;
907  int m;
908  int i;
909  int k;
910  unsigned long long max_bw;
911 
912  state = GNUNET_malloc (sizeof(double) * agent->m);
913 
914  max_bw = ril_get_max_bw((struct RIL_Scope *) agent->address_inuse->solver_information);
915 
916  y[0] = (double) agent->bw_out;
917  y[1] = (double) agent->bw_in;
918 
919  m = agent_address_get_index (agent, agent->address_inuse) * (solver->parameters.rbf_divisor+1) * (solver->parameters.rbf_divisor+1);
920  for (i = 0; i <= solver->parameters.rbf_divisor; i++)
921  {
922  for (k = 0; k <= solver->parameters.rbf_divisor; k++)
923  {
924  x[0] = (double) i * (double) max_bw / (double) solver->parameters.rbf_divisor;
925  x[1] = (double) k * (double) max_bw / (double) solver->parameters.rbf_divisor;
926  d[0] = x[0]-y[0];
927  d[1] = x[1]-y[1];
928  sigma = (((double) max_bw / ((double) solver->parameters.rbf_divisor + 1)) * 0.5);
929  f = exp(-((d[0]*d[0] + d[1]*d[1]) / (2 * sigma * sigma)));
930  state[m++] = f;
931  }
932  }
933 
934  return state;
935 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
unsigned int m
Number of rows of W / Number of state-vector features.
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:99
enum State state
current state of profiling
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
struct RIL_Learning_Parameters parameters
Learning parameters.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
unsigned long long rbf_divisor
State space divisor.
#define GNUNET_malloc(size)
Wrapper around malloc.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_utility()

static double agent_get_utility ( struct RIL_Peer_Agent agent)
static

Returns the utility value of the connection an agent manages.

Parameters
agentthe agent in question
Returns
the utility value

Definition at line 945 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, RIL_Peer_Agent::envi, GNUNET_ATS_PluginEnvironment::get_preferences, GNUNET_ATS_PREFERENCE_BANDWIDTH, GNUNET_ATS_PREFERENCE_LATENCY, GAS_NormalizationInfo::norm, ATS_Address::norm_delay, RIL_Peer_Agent::peer, RIL_MIN_BW, and RIL_UTILITY_DELAY_MAX.

Referenced by envi_get_reward(), and ril_network_get_social_welfare().

946 {
947  const double *preferences;
948  double delay_atsi;
949  double delay_norm;
950  double pref_match;
951 
952  preferences = agent->envi->env->get_preferences (agent->envi->env->cls,
953  &agent->peer);
954 
955  delay_atsi = agent->address_inuse->norm_delay.norm;
956  delay_norm = RIL_UTILITY_DELAY_MAX*exp(-delay_atsi*0.00001);
957 
958  pref_match = preferences[GNUNET_ATS_PREFERENCE_LATENCY] * delay_norm;
959  pref_match += preferences[GNUNET_ATS_PREFERENCE_BANDWIDTH] *
960  sqrt((double) (agent->bw_in/RIL_MIN_BW) * (double) (agent->bw_out/RIL_MIN_BW));
961  return pref_match;
962 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
#define RIL_UTILITY_DELAY_MAX
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
struct GNUNET_PeerIdentity peer
Peer ID.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Change the peer&#39;s latency value to the given amount.
Change the peer&#39;s bandwidth value (value per byte of bandwidth in the goal function) to the given amo...
struct GAS_NormalizationInfo norm_delay
Normalized delay information for this address.
void * cls
Closure to pass to all callbacks in this struct.
double norm
Normalized values from queue to a range of values [1.0...2.0].
struct GAS_RIL_Handle * envi
Environment handle.
#define RIL_MIN_BW
Here is the caller graph for this function:

◆ ril_network_get_social_welfare()

static double ril_network_get_social_welfare ( struct GAS_RIL_Handle solver,
struct RIL_Scope scope 
)
static

Calculates the social welfare within a network scope according to what social welfare measure is set in the configuration.

Parameters
solverthe solver handle
scopethe network scope in question
Returns
the social welfare value

Definition at line 973 of file plugin_ats_ril.c.

References RIL_Scope::active_agent_count, RIL_Peer_Agent::address_inuse, agent_get_utility(), GAS_RIL_Handle::agents_head, GNUNET_assert, GNUNET_MIN, GNUNET_NO, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, GAS_RIL_Handle::parameters, result, RIL_WELFARE_EGALITARIAN, RIL_WELFARE_NASH, RIL_Learning_Parameters::social_welfare, and ATS_Address::solver_information.

Referenced by ril_networks_update_state().

974 {
975  struct RIL_Peer_Agent *cur;
976  double result;
977 
978  switch (solver->parameters.social_welfare)
979  {
981  result = DBL_MAX;
982  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
983  {
984  if (cur->is_active && cur->address_inuse && (cur->address_inuse->solver_information == scope))
985  {
986  result = GNUNET_MIN(result, agent_get_utility(cur));
987  }
988  }
989  return result;
990 
991  case RIL_WELFARE_NASH:
992  result = 0;
993  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
994  {
995  if (cur->is_active && cur->address_inuse && (cur->address_inuse->solver_information == scope))
996  {
997  result *= pow(agent_get_utility(cur), 1.0 / (double) scope->active_agent_count);
998  }
999  }
1000  return result;
1001  }
1003  return 1;
1004 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:81
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:83
static int result
Global testing status.
struct RIL_Learning_Parameters parameters
Learning parameters.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
unsigned int active_agent_count
Number of active agents in scope.
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_get_penalty()

static double envi_get_penalty ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Definition at line 1007 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_INC, RIL_MIN_BW, and ATS_Address::solver_information.

Referenced by envi_get_reward().

1008 {
1009  struct RIL_Scope *net;
1010  unsigned long long over_max;
1011  unsigned long long over_in = 0;
1012  unsigned long long over_out = 0;
1013 
1014  net = agent->address_inuse->solver_information;
1015 
1016  if (net->bw_in_utilized > net->bw_in_available)
1017  {
1018  over_in = net->bw_in_utilized - net->bw_in_available;
1019  if (RIL_ACTION_BW_IN_INC == agent->a_old)
1020  {
1021  /* increase quadratically */
1022  over_in *= over_in;
1023  }
1024  }
1025  if (net->bw_out_utilized > net->bw_out_available)
1026  {
1027  over_out = net->bw_out_utilized - net->bw_out_available;
1028  if (RIL_ACTION_BW_OUT_INC == agent->a_old)
1029  {
1030  /* increase quadratically */
1031  over_out *= over_out;
1032  }
1033  }
1034  over_max = (over_in + over_out) / (RIL_MIN_BW * RIL_MIN_BW);
1035 
1036  return -1.0 * (double) over_max;
1037 }
int a_old
Last chosen action.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the caller graph for this function:

◆ envi_get_reward()

static double envi_get_reward ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Gets the reward for the last performed step, which is calculated in equal parts from the local (the peer specific) and the global (for all peers identical) reward.

Parameters
solverthe solver handle
agentthe agent handle
Returns
the reward

Definition at line 1049 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, agent_get_utility(), delta, envi_get_penalty(), RIL_Peer_Agent::nop_bonus, RIL_Peer_Agent::objective_old, RIL_ACTION_NOTHING, RIL_NOP_DECAY, RIL_Scope::social_welfare, and ATS_Address::solver_information.

Referenced by agent_step().

1050 {
1051  struct RIL_Scope *net;
1052  double objective;
1053  double delta;
1054  double steady;
1055  double penalty;
1056  double reward;
1057 
1058  net = agent->address_inuse->solver_information;
1059 
1060  penalty = envi_get_penalty(solver, agent);
1061  objective = (agent_get_utility (agent) + net->social_welfare) / 2;
1062  delta = objective - agent->objective_old;
1063  agent->objective_old = objective;
1064 
1065  if (delta != 0 && penalty == 0)
1066  {
1067  agent->nop_bonus = delta * RIL_NOP_DECAY;
1068  }
1069  else
1070  {
1071  agent->nop_bonus *= RIL_NOP_DECAY;
1072  }
1073 
1074  steady = (RIL_ACTION_NOTHING == agent->a_old) ? agent->nop_bonus : 0;
1075 
1076  reward = delta + steady;
1077  return reward + penalty;
1078 }
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
int a_old
Last chosen action.
double social_welfare
The social welfare achieved in the scope.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
#define RIL_NOP_DECAY
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
static double envi_get_penalty(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_double()

static void envi_action_bw_double ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Doubles the bandwidth for the active address.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise the outbound bandwidth

Definition at line 1088 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, ril_get_max_bw(), and ATS_Address::solver_information.

Referenced by envi_do_action().

1091 {
1092  unsigned long long new_bw;
1093  unsigned long long max_bw;
1094 
1095  max_bw = ril_get_max_bw((struct RIL_Scope *) agent->address_inuse->solver_information);
1096 
1097  if (direction_in)
1098  {
1099  new_bw = agent->bw_in * 2;
1100  if (new_bw < agent->bw_in || new_bw > max_bw)
1101  new_bw = max_bw;
1102  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1103  agent->bw_out, GNUNET_NO);
1104  }
1105  else
1106  {
1107  new_bw = agent->bw_out * 2;
1108  if (new_bw < agent->bw_out || new_bw > max_bw)
1109  new_bw = max_bw;
1110  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in,
1111  new_bw, GNUNET_NO);
1112  }
1113 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:81
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_halven()

static void envi_action_bw_halven ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Cuts the bandwidth for the active address in half.

The least amount of bandwidth suggested, is the minimum bandwidth for a peer, in order to not invoke a disconnect.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1125 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), and GNUNET_NO.

Referenced by envi_do_action().

1128 {
1129  unsigned long long new_bw;
1130 
1131  if (direction_in)
1132  {
1133  new_bw = agent->bw_in / 2;
1134  if (new_bw <= 0 || new_bw > agent->bw_in)
1135  new_bw = 0;
1136  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw, agent->bw_out,
1137  GNUNET_NO);
1138  }
1139  else
1140  {
1141  new_bw = agent->bw_out / 2;
1142  if (new_bw <= 0 || new_bw > agent->bw_out)
1143  new_bw = 0;
1144  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, new_bw,
1145  GNUNET_NO);
1146  }
1147 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:81
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_inc()

static void envi_action_bw_inc ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Increases the bandwidth by 5 times the minimum bandwidth for the active address.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1158 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, ril_get_max_bw(), RIL_INC_DEC_STEP_SIZE, RIL_MIN_BW, and ATS_Address::solver_information.

Referenced by envi_do_action().

1159 {
1160  unsigned long long new_bw;
1161  unsigned long long max_bw;
1162 
1163  max_bw = ril_get_max_bw((struct RIL_Scope *) agent->address_inuse->solver_information);
1164 
1165  if (direction_in)
1166  {
1167  new_bw = agent->bw_in + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1168  if (new_bw < agent->bw_in || new_bw > max_bw)
1169  new_bw = max_bw;
1170  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1171  agent->bw_out, GNUNET_NO);
1172  }
1173  else
1174  {
1175  new_bw = agent->bw_out + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1176  if (new_bw < agent->bw_out || new_bw > max_bw)
1177  new_bw = max_bw;
1178  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in,
1179  new_bw, GNUNET_NO);
1180  }
1181 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:81
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define RIL_INC_DEC_STEP_SIZE
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_dec()

static void envi_action_bw_dec ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Decreases the bandwidth by 5 times the minimum bandwidth for the active address.

The least amount of bandwidth suggested, is the minimum bandwidth for a peer, in order to not invoke a disconnect.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1193 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, RIL_INC_DEC_STEP_SIZE, and RIL_MIN_BW.

Referenced by envi_do_action().

1194 {
1195  unsigned long long new_bw;
1196 
1197  if (direction_in)
1198  {
1199  new_bw = agent->bw_in - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1200  if (new_bw <= 0 || new_bw > agent->bw_in)
1201  new_bw = 0;
1202  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw, agent->bw_out,
1203  GNUNET_NO);
1204  }
1205  else
1206  {
1207  new_bw = agent->bw_out - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1208  if (new_bw <= 0 || new_bw > agent->bw_out)
1209  new_bw = 0;
1210  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, new_bw,
1211  GNUNET_NO);
1212  }
1213 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:81
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define RIL_INC_DEC_STEP_SIZE
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_address_switch()

static void envi_action_address_switch ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
unsigned int  address_index 
)
static

Switches to the address given by its index.

Parameters
solversolver handle
agentagent handle
address_indexindex of the address as it is saved in the agent's list, starting with zero

Definition at line 1223 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_assert, GNUNET_NO, and RIL_Address_Wrapped::next.

Referenced by envi_do_action().

1226 {
1227  struct RIL_Address_Wrapped *cur;
1228  int i = 0;
1229 
1230  //cur = agent_address_get_wrapped(agent, agent->address_inuse);
1231 
1232  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
1233  {
1234  if (i == address_index)
1235  {
1236  envi_set_active_suggestion (solver, agent, cur->address_naked, agent->bw_in, agent->bw_out,
1237  GNUNET_NO);
1238  return;
1239  }
1240 
1241  i++;
1242  }
1243 
1244  //no address with address_index exists, in this case this action should not be callable
1246 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
#define GNUNET_NO
Definition: gnunet_common.h:81
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_do_action()

static void envi_do_action ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  action 
)
static

Puts the action into effect by calling the according function.

Parameters
solverthe solver handle
agentthe action handle
actionthe action to perform by the solver

Definition at line 1256 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), envi_action_address_switch(), envi_action_bw_dec(), envi_action_bw_double(), envi_action_bw_halven(), envi_action_bw_inc(), GNUNET_assert, GNUNET_NO, GNUNET_YES, RIL_ACTION_BW_IN_DBL, RIL_ACTION_BW_IN_DEC, RIL_ACTION_BW_IN_HLV, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_DBL, RIL_ACTION_BW_OUT_DEC, RIL_ACTION_BW_OUT_HLV, RIL_ACTION_BW_OUT_INC, RIL_ACTION_NOTHING, and RIL_ACTION_TYPE_NUM.

Referenced by agent_step().

1257 {
1258  int address_index;
1259 
1260  switch (action)
1261  {
1262  case RIL_ACTION_NOTHING:
1263  break;
1264  case RIL_ACTION_BW_IN_DBL:
1265  envi_action_bw_double (solver, agent, GNUNET_YES);
1266  break;
1267  case RIL_ACTION_BW_IN_HLV:
1268  envi_action_bw_halven (solver, agent, GNUNET_YES);
1269  break;
1270  case RIL_ACTION_BW_IN_INC:
1271  envi_action_bw_inc (solver, agent, GNUNET_YES);
1272  break;
1273  case RIL_ACTION_BW_IN_DEC:
1274  envi_action_bw_dec (solver, agent, GNUNET_YES);
1275  break;
1276  case RIL_ACTION_BW_OUT_DBL:
1277  envi_action_bw_double (solver, agent, GNUNET_NO);
1278  break;
1279  case RIL_ACTION_BW_OUT_HLV:
1280  envi_action_bw_halven (solver, agent, GNUNET_NO);
1281  break;
1282  case RIL_ACTION_BW_OUT_INC:
1283  envi_action_bw_inc (solver, agent, GNUNET_NO);
1284  break;
1285  case RIL_ACTION_BW_OUT_DEC:
1286  envi_action_bw_dec (solver, agent, GNUNET_NO);
1287  break;
1288  default:
1289  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) //switch address action
1290  {
1291  address_index = action - RIL_ACTION_TYPE_NUM;
1292 
1293  GNUNET_assert(address_index >= 0);
1294  GNUNET_assert(
1295  address_index <= agent_address_get_index (agent, agent->addresses_tail->address_naked));
1296 
1297  envi_action_address_switch (solver, agent, address_index);
1298  break;
1299  }
1300  // error - action does not exist
1302  }
1303 }
static void envi_action_bw_halven(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Cuts the bandwidth for the active address in half.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void envi_action_bw_dec(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Decreases the bandwidth by 5 times the minimum bandwidth for the active address.
#define GNUNET_NO
Definition: gnunet_common.h:81
struct ATS_Address * address_naked
The address.
static void envi_action_address_switch(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
Switches to the address given by its index.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
static void envi_action_bw_double(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Doubles the bandwidth for the active address.
#define GNUNET_YES
Definition: gnunet_common.h:80
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
static void envi_action_bw_inc(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Increases the bandwidth by 5 times the minimum bandwidth for the active address.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_egreedy()

static int agent_select_egreedy ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Selects the next action using the e-greedy strategy.

I.e. with a probability of (1-e) the action with the maximum expected return will be chosen (=> exploitation) and with probability (e) a random action will be chosen. In case the Q-learning rule is set, the function also resets the eligibility traces in the exploration case (after Watkin's Q-learning).

Parameters
agentthe agent selecting an action
statethe current state-feature vector
Returns
the action index

Definition at line 1317 of file plugin_ats_ril.c.

References agent_get_action_max(), agent_get_action_random(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_decay, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_YES, GAS_RIL_Handle::parameters, and RIL_ALGO_Q.

Referenced by agent_select_action().

1318 {
1319  int action;
1321  UINT32_MAX) / (double) UINT32_MAX;
1322 
1323  if (r < agent->envi->parameters.epsilon) //explore
1324  {
1325  action = agent_get_action_random(agent);
1326  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1327  {
1328  agent->eligibility_reset = GNUNET_YES;
1329  }
1330  agent->envi->parameters.epsilon *= agent->envi->parameters.epsilon_decay;
1331  return action;
1332  }
1333  else //exploit
1334  {
1335  action = agent_get_action_max(agent, state);
1336  return action;
1337  }
1338 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int agent_get_action_random(struct RIL_Peer_Agent *agent)
Chooses a random action from the set of possible ones.
enum State state
current state of profiling
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_decay
Decay factor of the explore ratio.
#define GNUNET_YES
Definition: gnunet_common.h:80
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_softmax()

static int agent_select_softmax ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Selects the next action with a probability corresponding to its value.

The probability is calculated using a Boltzmann distribution with a temperature value. The higher the temperature, the more are the action selection probabilities the same. With a temperature of 0, the selection is greedy, i.e. always the action with the highest value is chosen.

Parameters
agent
state
Returns

Definition at line 1351 of file plugin_ats_ril.c.

References agent_action_is_possible(), agent_get_action_max(), agent_q(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, GNUNET_assert, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::n, p, GAS_RIL_Handle::parameters, RIL_ACTION_INVALID, RIL_ALGO_Q, RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_decay.

Referenced by agent_select_action().

1352 {
1353  int i;
1354  int a_max;
1355  double eqt[agent->n];
1356  double p[agent->n];
1357  double sum = 0;
1358  double r;
1359 
1360  a_max = agent_get_action_max(agent, state);
1361 
1362  for (i=0; i<agent->n; i++)
1363  {
1364  if (agent_action_is_possible(agent, i))
1365  {
1366  eqt[i] = exp(agent_q(agent,state,i) / agent->envi->parameters.temperature);
1367  if (isinf (eqt[i]))
1368  eqt[i] = isinf(eqt[i]) * UINT32_MAX;
1369  sum += eqt[i];
1370  }
1371  }
1372  for (i=0; i<agent->n; i++)
1373  {
1374  if (agent_action_is_possible(agent, i))
1375  {
1376  p[i] = eqt[i]/sum;
1377  }
1378  else
1379  {
1380  p[i] = 0;
1381  }
1382  }
1384  UINT32_MAX) / (double) UINT32_MAX;
1385  sum = 0;
1386  for (i=0; i<agent->n; i++)
1387  {
1388  if (sum + p[i] > r)
1389  {
1390  if (i != a_max)
1391  {
1392  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1393  agent->eligibility_reset = GNUNET_YES;
1395  }
1396  return i;
1397  }
1398  sum += p[i];
1399  }
1401  return RIL_ACTION_INVALID;
1402 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double temperature_decay
Decay factor of the temperature value.
#define GNUNET_NO
Definition: gnunet_common.h:81
double temperature
Softmax action-selection temperature.
enum State state
current state of profiling
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_YES
Definition: gnunet_common.h:80
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_action()

static int agent_select_action ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Select the next action of an agent either according to the e-greedy strategy or the softmax strategy.

Parameters
agentthe agent in question
statethe current state-feature vector
Returns
the action index

Definition at line 1413 of file plugin_ats_ril.c.

References agent_select_egreedy(), agent_select_softmax(), RIL_Peer_Agent::envi, GAS_RIL_Handle::parameters, RIL_SELECT_EGREEDY, and RIL_Learning_Parameters::select.

Referenced by agent_step().

1414 {
1415  if (agent->envi->parameters.select == RIL_SELECT_EGREEDY)
1416  {
1417  return agent_select_egreedy(agent, state);
1418  }
1419  else
1420  {
1421  return agent_select_softmax(agent, state);
1422  }
1423 }
static int agent_select_softmax(struct RIL_Peer_Agent *agent, double *state)
Selects the next action with a probability corresponding to its value.
enum State state
current state of profiling
enum RIL_Select select
Action selection strategy;.
static int agent_select_egreedy(struct RIL_Peer_Agent *agent, double *state)
Selects the next action using the e-greedy strategy.
struct RIL_Learning_Parameters parameters
Learning parameters.
struct GAS_RIL_Handle * envi
Environment handle.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_step()

static void agent_step ( struct RIL_Peer_Agent agent)
static

Performs one step of the Markov Decision Process.

Other than in the literature the step starts after having done the last action a_old. It observes the new state s_next and the reward received. Then the coefficient update is done according to the SARSA or Q-learning method. The next action is put into effect.

Parameters
agentthe agent performing the step

Definition at line 1434 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, agent_get_action_max(), agent_modify_eligibility(), agent_select_action(), agent_update(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::eligibility_reset, RIL_Learning_Parameters::eligibility_trace_mode, RIL_Peer_Agent::envi, envi_do_action(), envi_get_reward(), envi_get_state(), GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_i2s(), GNUNET_NO, LOG, GAS_RIL_Handle::parameters, RIL_Peer_Agent::peer, RIL_ACTION_INVALID, RIL_ALGO_Q, RIL_ALGO_SARSA, RIL_E_DECAY, RIL_E_ZERO, ril_step(), RIL_Peer_Agent::s_old, and RIL_Peer_Agent::step_count.

Referenced by ril_step().

1435 {
1436  int a_next = RIL_ACTION_INVALID;
1437  int a_max;
1438  double *s_next;
1439  double reward;
1440 
1441  LOG(GNUNET_ERROR_TYPE_DEBUG, " agent_step() Peer '%s', algorithm %s\n",
1442  GNUNET_i2s (&agent->peer),
1443  agent->envi->parameters.algorithm ? "Q" : "SARSA");
1444 
1445  s_next = envi_get_state (agent->envi, agent);
1446  reward = envi_get_reward (agent->envi, agent);
1447 
1448  if (agent->eligibility_reset)
1449  {
1450  agent_modify_eligibility(agent, RIL_E_ZERO, NULL, -1);
1451  agent->eligibility_reset = GNUNET_NO;
1452  }
1453  else
1454  {
1455  agent_modify_eligibility (agent, RIL_E_DECAY, NULL, -1);
1456  }
1457  if (RIL_ACTION_INVALID != agent->a_old)
1458  {
1459  agent_modify_eligibility (agent, agent->envi->parameters.eligibility_trace_mode, agent->s_old, agent->a_old);
1460  }
1461 
1462  switch (agent->envi->parameters.algorithm)
1463  {
1464  case RIL_ALGO_SARSA:
1465  a_next = agent_select_action (agent, s_next);
1466  if (RIL_ACTION_INVALID != agent->a_old)
1467  {
1468  //updates weights with selected action (on-policy), if not first step
1469  agent_update (agent, reward, s_next, a_next);
1470  }
1471  break;
1472 
1473  case RIL_ALGO_Q:
1474  a_max = agent_get_action_max (agent, s_next);
1475  if (RIL_ACTION_INVALID != agent->a_old)
1476  {
1477  //updates weights with best action, disregarding actually selected action (off-policy), if not first step
1478  agent_update (agent, reward, s_next, a_max);
1479  }
1480  a_next = agent_select_action (agent, s_next);
1481  break;
1482  }
1483 
1484  GNUNET_assert(RIL_ACTION_INVALID != a_next);
1485 
1486  LOG (GNUNET_ERROR_TYPE_DEBUG, "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n",
1487  agent->step_count,
1488  reward,
1489  agent->bw_in/1024,
1490  agent->bw_out/1024,
1491  a_next);
1492 
1493  envi_do_action (agent->envi, agent, a_next);
1494 
1495  GNUNET_free(agent->s_old);
1496  agent->s_old = s_next;
1497  agent->a_old = a_next;
1498 
1499  agent->step_count += 1;
1500 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
int a_old
Last chosen action.
#define GNUNET_NO
Definition: gnunet_common.h:81
static double * envi_get_state(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Allocates a state vector and fills it with the features present.
struct GNUNET_PeerIdentity peer
Peer ID.
static void envi_do_action(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
Puts the action into effect by calling the according function.
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
#define RIL_ACTION_INVALID
static void agent_update(struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
Updates the weights (i.e.
static double envi_get_reward(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Gets the reward for the last performed step, which is calculated in equal parts from the local (the p...
static int agent_select_action(struct RIL_Peer_Agent *agent, double *state)
Select the next action of an agent either according to the e-greedy strategy or the softmax strategy...
double * s_old
Last perceived state feature vector.
#define LOG(kind,...)
static void agent_modify_eligibility(struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each co...
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
struct GAS_RIL_Handle * envi
Environment handle.
#define GNUNET_free(ptr)
Wrapper around free.
unsigned long long step_count
Number of performed time-steps.
enum RIL_E_Modification eligibility_trace_mode
Whether to accumulate or replace eligibility traces.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step()

static void ril_step ( struct GAS_RIL_Handle solver)
static

Prototype of the ril_step() procedure.

Triggers one step per agent.

Parameters
solverthe solver handle
solver

Definition at line 1864 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, agent_step(), GAS_RIL_Handle::agents_head, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, GAS_RIL_Handle::bulk_changes, GAS_RIL_Handle::bulk_lock, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, GAS_OP_SOLVE_START, GAS_OP_SOLVE_STOP, GAS_OP_SOLVE_UPDATE_NOTIFICATION_START, GAS_OP_SOLVE_UPDATE_NOTIFICATION_STOP, GAS_STAT_SUCCESS, GNUNET_ERROR_TYPE_DEBUG, GNUNET_NO, GNUNET_TIME_absolute_get(), GNUNET_YES, RIL_Peer_Agent::is_active, LOG, RIL_Peer_Agent::next, ril_calculate_discount(), ril_inform(), ril_networks_update_state(), ril_step_schedule_next(), ril_try_unblock_agent(), GAS_RIL_Handle::step_count, GAS_RIL_Handle::step_time_last, RIL_Peer_Agent::suggestion_address, and RIL_Peer_Agent::suggestion_issue.

Referenced by agent_step(), GAS_ril_address_add(), GAS_ril_address_change_preference(), GAS_ril_address_delete(), GAS_ril_address_property_changed(), GAS_ril_bulk_stop(), GAS_ril_stop_get_preferred_address(), and ril_step_scheduler_task().

1865 {
1866  struct RIL_Peer_Agent *cur;
1867 
1868  if (GNUNET_YES == solver->bulk_lock)
1869  {
1870  solver->bulk_changes++;
1871  return;
1872  }
1873 
1875 
1876  LOG(GNUNET_ERROR_TYPE_DEBUG, " RIL step number %d\n", solver->step_count);
1877 
1878  if (0 == solver->step_count)
1879  {
1881  }
1882 
1883  ril_calculate_discount (solver);
1884  ril_networks_update_state (solver);
1885 
1886  //trigger one step per active, unblocked agent
1887  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1888  {
1889  if (cur->is_active)
1890  {
1891  if (NULL == cur->address_inuse)
1892  {
1893  ril_try_unblock_agent(solver, cur, GNUNET_NO);
1894  }
1895  if (cur->address_inuse)
1896  {
1897  agent_step (cur);
1898  }
1899  }
1900  }
1901 
1902  ril_networks_update_state (solver);
1903 
1904  solver->step_count++;
1905  ril_step_schedule_next (solver);
1906 
1908 
1910  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1911  {
1912  if (cur->suggestion_issue) {
1913  solver->env->bandwidth_changed_cb (solver->env->cls,
1914  cur->suggestion_address);
1915  cur->suggestion_issue = GNUNET_NO;
1916  }
1917  }
1919 }
After the problem was finished, start notifications about changes to addresses.
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
int bulk_changes
Number of changes during a lock.
static void ril_inform(struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
Informs the environment about the status of the solver.
#define GNUNET_NO
Definition: gnunet_common.h:81
int suggestion_issue
Flag whether a suggestion has to be issued.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
unsigned long long step_count
Number of performed steps.
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
A solution iteration has been started.
static void ril_step_schedule_next(struct GAS_RIL_Handle *solver)
Schedules the next global step in an adaptive way.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
static void ril_calculate_discount(struct GAS_RIL_Handle *solver)
Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.
struct ATS_Address * address_inuse
Address in use.
A solution iteration has been finished.
static void agent_step(struct RIL_Peer_Agent *agent)
Performs one step of the Markov Decision Process.
static void ril_networks_update_state(struct GAS_RIL_Handle *solver)
Retrieves the state of the network scope, so that its attributes are up-to- date. ...
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:80
int bulk_lock
Lock for bulk operations.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
#define LOG(kind,...)
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
After the problem was finished, notifications about changes to addresses are done.
struct ATS_Address * suggestion_address
The address which has to be issued.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step_scheduler_task()

static void ril_step_scheduler_task ( void *  cls)
static

Task for the scheduler, which performs one step and lets the solver know that no further step is scheduled.

Parameters
clsthe solver handle

Definition at line 1518 of file plugin_ats_ril.c.

References ril_step(), and GAS_RIL_Handle::step_next_task_id.

Referenced by ril_step_schedule_next().

1519 {
1520  struct GAS_RIL_Handle *solver = cls;
1521 
1522  solver->step_next_task_id = NULL;
1523  ril_step (solver);
1524 }
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
A handle for the reinforcement learning solver.
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_get_used_resource_ratio()

static double ril_get_used_resource_ratio ( struct GAS_RIL_Handle solver)
static

Determines how much of the available bandwidth is assigned.

If more is assigned than available it returns 1. The function is used to determine the step size of the adaptive stepping.

Parameters
solverthe solver handle
Returns
the ratio

Definition at line 1535 of file plugin_ats_ril.c.

References RIL_Scope::bw_in_assigned, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, GAS_RIL_Handle::network_entries, and GAS_RIL_Handle::networks_count.

Referenced by ril_step_schedule_next().

1536 {
1537  int i;
1538  struct RIL_Scope net;
1539  unsigned long long sum_assigned = 0;
1540  unsigned long long sum_available = 0;
1541  double ratio;
1542 
1543  for (i = 0; i < solver->networks_count; i++)
1544  {
1545  net = solver->network_entries[i];
1546  if (net.bw_in_assigned > 0) //only consider scopes where an address is actually active
1547  {
1548  sum_assigned += net.bw_in_utilized;
1549  sum_assigned += net.bw_out_utilized;
1550  sum_available += net.bw_in_available;
1551  sum_available += net.bw_out_available;
1552  }
1553  }
1554  if (sum_available > 0)
1555  {
1556  ratio = ((double) sum_assigned) / ((double) sum_available);
1557  }
1558  else
1559  {
1560  ratio = 0;
1561  }
1562 
1563  return ratio > 1 ? 1 : ratio; //overutilization is possible, cap at 1
1564 }
struct RIL_Scope * network_entries
Array of networks with global assignment state.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
unsigned int networks_count
Networks count.
Here is the caller graph for this function:

◆ ril_get_network()

static struct RIL_Scope* ril_get_network ( struct GAS_RIL_Handle s,
uint32_t  type 
)
static

Lookup network struct by type.

Parameters
sthe solver handle
typethe network type
Returns
the network struct

Definition at line 1574 of file plugin_ats_ril.c.

References GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, and RIL_Scope::type.

Referenced by GAS_ril_address_add(), ril_network_is_active(), and ril_network_is_not_full().

1575 {
1576  int i;
1577 
1578  for (i = 0; i < s->networks_count; i++)
1579  {
1580  if (s->network_entries[i].type == type)
1581  {
1582  return &s->network_entries[i];
1583  }
1584  }
1585  return NULL ;
1586 }
struct RIL_Scope * network_entries
Array of networks with global assignment state.
enum GNUNET_NetworkType type
ATS network type.
unsigned int networks_count
Networks count.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Here is the caller graph for this function:

◆ ril_network_is_not_full()

static int ril_network_is_not_full ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  network 
)
static

Determines whether more connections are allocated in a network scope, than they would theoretically fit.

This is used as a heuristic to determine, whether a new connection can be allocated or not.

Parameters
solverthe solver handle
networkthe network scope in question
Returns
GNUNET_YES if there are theoretically enough resources left

Definition at line 1598 of file plugin_ats_ril.c.

References address_count, RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, ril_get_network(), RIL_MIN_BW, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_try_unblock_agent().

1599 {
1600  struct RIL_Scope *net;
1601  struct RIL_Peer_Agent *agent;
1602  unsigned long long address_count = 0;
1603 
1604  for (agent = solver->agents_head; NULL != agent; agent = agent->next)
1605  {
1606  if (agent->address_inuse && agent->is_active)
1607  {
1608  net = agent->address_inuse->solver_information;
1609  if (net->type == network)
1610  {
1611  address_count++;
1612  }
1613  }
1614  }
1615 
1616  net = ril_get_network (solver, network);
1617  return (net->bw_in_available > RIL_MIN_BW * address_count) && (net->bw_out_available > RIL_MIN_BW * address_count);
1618 }
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
static int address_count
Definition: gnunet-hello.c:50
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_try_unblock_agent()

static void ril_try_unblock_agent ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  silent 
)
static

Unblocks an agent for which a connection request is there, that could not be satisfied.

Iterates over the addresses of the agent, if one of its addresses can now be allocated in its scope the agent is unblocked, otherwise it remains unchanged.

Parameters
solverthe solver handle
agentthe agent in question
silent

Definition at line 1631 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, envi_set_active_suggestion(), RIL_Address_Wrapped::next, RIL_MIN_BW, ril_network_is_not_full(), ATS_Address::solver_information, and RIL_Scope::type.

Referenced by GAS_ril_address_add(), GAS_ril_get_preferred_address(), and ril_step().

1632 {
1633  struct RIL_Address_Wrapped *addr_wrap;
1634  struct RIL_Scope *net;
1635  unsigned long long start_in;
1636  unsigned long long start_out;
1637 
1638  for (addr_wrap = agent->addresses_head; NULL != addr_wrap; addr_wrap = addr_wrap->next)
1639  {
1640  net = addr_wrap->address_naked->solver_information;
1641  if (ril_network_is_not_full(solver, net->type))
1642  {
1643  if (NULL == agent->address_inuse)
1644  {
1645  start_in = net->bw_in_available < net->bw_in_utilized ? (net->bw_in_available - net->bw_in_utilized) / 2 : RIL_MIN_BW;
1646  start_out = net->bw_out_available < net->bw_out_utilized ? (net->bw_out_available - net->bw_out_utilized) / 2 : RIL_MIN_BW;
1647  envi_set_active_suggestion (solver, agent, addr_wrap->address_naked, start_in, start_out, silent);
1648  }
1649  return;
1650  }
1651  }
1652  agent->address_inuse = NULL;
1653 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
static int ril_network_is_not_full(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determines whether more connections are allocated in a network scope, than they would theoretically f...
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
static int silent
–silent option
Definition: gnunet-qr.c:49
enum GNUNET_NetworkType type
ATS network type.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_calculate_discount()

static void ril_calculate_discount ( struct GAS_RIL_Handle solver)
static

Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.

Parameters
solverthe solver handle

Definition at line 1662 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::beta, RIL_Learning_Parameters::gamma, GAS_RIL_Handle::global_discount_integrated, GAS_RIL_Handle::global_discount_variable, GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_difference(), GAS_RIL_Handle::parameters, GNUNET_TIME_Relative::rel_value_us, GAS_RIL_Handle::simulate, GAS_RIL_Handle::step_time_last, and RIL_Learning_Parameters::step_time_min.

Referenced by ril_step().

1663 {
1664  struct GNUNET_TIME_Absolute time_now;
1665  struct GNUNET_TIME_Relative time_delta;
1666  double tau;
1667 
1668  // MDP case only for debugging purposes
1669  if (solver->simulate)
1670  {
1671  solver->global_discount_variable = solver->parameters.gamma;
1672  solver->global_discount_integrated = 1;
1673  return;
1674  }
1675 
1676  // semi-MDP case
1677 
1678  //calculate tau, i.e. how many real valued time units have passed, one time unit is one minimum time step
1679  time_now = GNUNET_TIME_absolute_get ();
1680  time_delta = GNUNET_TIME_absolute_get_difference (solver->step_time_last, time_now);
1681  solver->step_time_last = time_now;
1682  tau = (double) time_delta.rel_value_us
1683  / (double) solver->parameters.step_time_min.rel_value_us;
1684 
1685  //calculate reward discounts (once per step for all agents)
1686  solver->global_discount_variable = pow (M_E, ((-1.0) * ((double) solver->parameters.beta) * tau));
1687  solver->global_discount_integrated = (1.0 - solver->global_discount_variable)
1688  / (double) solver->parameters.beta;
1689 }
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
uint64_t rel_value_us
The actual value.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
unsigned long long simulate
Simulate steps, i.e.
double beta
Learning discount variable in the TD-update for semi-MDPs.
struct RIL_Learning_Parameters parameters
Learning parameters.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
double global_discount_variable
Variable discount factor, dependent on time between steps.
Time for absolute times used by GNUnet, in microseconds.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_difference(struct GNUNET_TIME_Absolute start, struct GNUNET_TIME_Absolute end)
Compute the time difference between the given start and end times.
Definition: time.c:353
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
Time for relative time used by GNUnet, in microseconds.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_network_count_active_agents()

static int ril_network_count_active_agents ( struct GAS_RIL_Handle solver,
struct RIL_Scope scope 
)
static

Count the number of active agents/connections in a network scope.

Parameters
solverthe solver handle
scopethe network scope in question
Returns
the number of allocated connections

Definition at line 1699 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, and ATS_Address::solver_information.

Referenced by ril_networks_update_state().

1700 {
1701  int c = 0;
1702  struct RIL_Peer_Agent *cur_agent;
1703 
1704  for (cur_agent = solver->agents_head; NULL != cur_agent; cur_agent = cur_agent->next)
1705  {
1706  if (cur_agent->is_active && cur_agent->address_inuse && (cur_agent->address_inuse->solver_information == scope))
1707  {
1708  c++;
1709  }
1710  }
1711  return c;
1712 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the caller graph for this function:

◆ ril_network_get_assigned()

static unsigned long long ril_network_get_assigned ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  type,
int  direction_in 
)
static

Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the outbound direction.

Parameters
solverthe solver handle
typethe type of the network scope in question
direction_inGNUNET_YES if the inbound direction should be summed up, otherwise the outbound direction will be summed up
Returns
the sum of the assigned bandwidths

Definition at line 1725 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_networks_update_state().

1726 {
1727  struct RIL_Peer_Agent *cur;
1728  struct RIL_Scope *net;
1729  unsigned long long sum = 0;
1730 
1731  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1732  {
1733  if (cur->is_active && cur->address_inuse)
1734  {
1735  net = cur->address_inuse->solver_information;
1736  if (net->type == type)
1737  {
1738  if (direction_in)
1739  sum += cur->bw_in;
1740  else
1741  sum += cur->bw_out;
1742  }
1743  }
1744  }
1745 
1746  return sum;
1747 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the caller graph for this function:

◆ ril_network_get_utilized()

static unsigned long long ril_network_get_utilized ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  type,
int  direction_in 
)
static

Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound or in the outbound direction.

Parameters
solverthe solver handle
typethe type of the network scope in question
direction_inGNUNET_YES if the inbound direction should be summed up, otherwise the outbound direction will be summed up
Returns
the sum of the utilized bandwidths (in bytes/second)

Definition at line 1760 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, GAS_NormalizationInfo::norm, ATS_Address::norm_utilization_in, ATS_Address::norm_utilization_out, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_networks_update_state().

1761 {
1762  struct RIL_Peer_Agent *cur;
1763  struct RIL_Scope *net;
1764  unsigned long long sum = 0;
1765 
1766  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1767  {
1768  if (cur->is_active && cur->address_inuse)
1769  {
1770  net = cur->address_inuse->solver_information;
1771  if (net->type == type)
1772  {
1773  if (direction_in)
1774  sum += cur->address_inuse->norm_utilization_in.norm;
1775  else
1777  }
1778  }
1779  }
1780 
1781  return sum;
1782 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
struct GAS_NormalizationInfo norm_utilization_out
Normalized utilization outbound for this address.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
struct GAS_NormalizationInfo norm_utilization_in
Normalized utilization inbound for this address.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
double norm
Normalized values from queue to a range of values [1.0...2.0].
Here is the caller graph for this function:

◆ ril_networks_update_state()

static void ril_networks_update_state ( struct GAS_RIL_Handle solver)
static

Retrieves the state of the network scope, so that its attributes are up-to- date.

Parameters
solverthe solver handle

Definition at line 1791 of file plugin_ats_ril.c.

References RIL_Scope::active_agent_count, RIL_Scope::bw_in_assigned, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_assigned, RIL_Scope::bw_out_utilized, GNUNET_NO, GNUNET_YES, GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, ril_network_count_active_agents(), ril_network_get_assigned(), ril_network_get_social_welfare(), ril_network_get_utilized(), RIL_Scope::social_welfare, and RIL_Scope::type.

Referenced by ril_step().

1792 {
1793  int c;
1794  struct RIL_Scope *net;
1795 
1796  for (c = 0; c < solver->networks_count; c++)
1797  {
1798  net = &solver->network_entries[c];
1804  net->social_welfare = ril_network_get_social_welfare(solver, net);
1805  }
1806 }
#define GNUNET_NO
Definition: gnunet_common.h:81
struct RIL_Scope * network_entries
Array of networks with global assignment state.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
enum GNUNET_NetworkType type
ATS network type.
static unsigned long long ril_network_get_utilized(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound o...
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
double social_welfare
The social welfare achieved in the scope.
static int ril_network_count_active_agents(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Count the number of active agents/connections in a network scope.
unsigned int networks_count
Networks count.
static unsigned long long ril_network_get_assigned(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the ...
unsigned int active_agent_count
Number of active agents in scope.
uint32_t bw_in_assigned
Bandwidth inbound assigned in network after last step.
unsigned long long bw_out_assigned
Bandwidth outbound assigned in network after last step.
#define GNUNET_YES
Definition: gnunet_common.h:80
static double ril_network_get_social_welfare(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Calculates the social welfare within a network scope according to what social welfare measure is set ...
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step_schedule_next()

static void ril_step_schedule_next ( struct GAS_RIL_Handle solver)
static

Schedules the next global step in an adaptive way.

The more resources are left, the earlier the next step is scheduled. This serves the reactivity of the solver to changed inputs.

Parameters
solverthe solver handle

Definition at line 1816 of file plugin_ats_ril.c.

References GAS_RIL_Handle::done, GNUNET_assert, GNUNET_NO, GNUNET_SCHEDULER_add_delayed(), GNUNET_TIME_relative_saturating_multiply(), GNUNET_TIME_relative_subtract(), GNUNET_TIME_UNIT_MICROSECONDS, GNUNET_TIME_UNIT_ZERO, GAS_RIL_Handle::parameters, GNUNET_TIME_Relative::rel_value_us, ril_get_used_resource_ratio(), RIL_INTERVAL_EXPONENT, ril_step_scheduler_task(), GAS_RIL_Handle::simulate, GAS_RIL_Handle::step_next_task_id, RIL_Learning_Parameters::step_time_max, and RIL_Learning_Parameters::step_time_min.

Referenced by ril_step().

1817 {
1818  double used_ratio;
1819  double factor;
1820  double y;
1821  double offset;
1822  struct GNUNET_TIME_Relative time_next;
1823 
1824  used_ratio = ril_get_used_resource_ratio (solver);
1825 
1826  GNUNET_assert(
1829 
1830  factor = (double) GNUNET_TIME_relative_subtract (solver->parameters.step_time_max,
1832  offset = (double) solver->parameters.step_time_min.rel_value_us;
1833  y = factor * pow (used_ratio, RIL_INTERVAL_EXPONENT) + offset;
1834 
1835  GNUNET_assert(y <= (double) solver->parameters.step_time_max.rel_value_us);
1836  GNUNET_assert(y >= (double) solver->parameters.step_time_min.rel_value_us);
1837 
1838  time_next = GNUNET_TIME_relative_saturating_multiply (GNUNET_TIME_UNIT_MICROSECONDS, (unsigned long long) y);
1839 
1840 // LOG (GNUNET_ERROR_TYPE_INFO, "ratio: %f, factor: %f, offset: %f, y: %f\n",
1841 // used_ratio,
1842 // factor,
1843 // offset,
1844 // y);
1845 
1846  if (solver->simulate)
1847  {
1848  time_next = GNUNET_TIME_UNIT_ZERO;
1849  }
1850 
1851  if ((NULL == solver->step_next_task_id) && (GNUNET_NO == solver->done))
1852  {
1854  solver);
1855  }
1856 }
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
uint64_t rel_value_us
The actual value.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define GNUNET_NO
Definition: gnunet_common.h:81
#define RIL_INTERVAL_EXPONENT
static void ril_step_scheduler_task(void *cls)
Task for the scheduler, which performs one step and lets the solver know that no further step is sche...
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1246
unsigned long long simulate
Simulate steps, i.e.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_subtract(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Subtract relative timestamp from the other.
Definition: time.c:601
struct RIL_Learning_Parameters parameters
Learning parameters.
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
int done
Shutdown.
static double ril_get_used_resource_ratio(struct GAS_RIL_Handle *solver)
Determines how much of the available bandwidth is assigned.
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
#define GNUNET_TIME_UNIT_MICROSECONDS
One microsecond, our basic time unit.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:499
Time for relative time used by GNUnet, in microseconds.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_w_init()

static void agent_w_init ( struct RIL_Peer_Agent agent)
static

Initializes the matrix W of parameter vectors theta with small random numbers.

Parameters
agentThe respective agent

Definition at line 1927 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::alpha, RIL_Peer_Agent::envi, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, and RIL_Peer_Agent::W.

Referenced by agent_init().

1928 {
1929  int i;
1930  int k;
1931 
1932  for (i = 0; i < agent->n; i++)
1933  {
1934  for (k = 0; k < agent->m; k++)
1935  {
1936  agent->W[i][k] = agent->envi->parameters.alpha * (1.0 - 2.0 * ((double) GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX)/(double) UINT32_MAX));
1937  }
1938  }
1939 }
unsigned int m
Number of rows of W / Number of state-vector features.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
unsigned int n
Number of columns of W / Number of actions.
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
double alpha
Gradient-descent step-size.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_init()

static struct RIL_Peer_Agent* agent_init ( void *  s,
const struct GNUNET_PeerIdentity peer 
)
static

Initialize an agent without addresses and its knowledge base.

Parameters
sril solver
peerthe one in question
Returns
handle to the new agent

Definition at line 1949 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, agent_w_init(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::E, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, GNUNET_malloc, GNUNET_new, GNUNET_NO, RIL_Peer_Agent::is_active, RIL_Peer_Agent::m, RIL_Peer_Agent::n, RIL_Peer_Agent::nop_bonus, RIL_Peer_Agent::objective_old, peer, RIL_Peer_Agent::peer, RIL_ACTION_INVALID, RIL_ACTION_TYPE_NUM, RIL_MIN_BW, RIL_Peer_Agent::s_old, RIL_Peer_Agent::step_count, RIL_Peer_Agent::suggestion_issue, and RIL_Peer_Agent::W.

Referenced by ril_get_agent().

1950 {
1951  int i;
1952  struct GAS_RIL_Handle * solver = s;
1953  struct RIL_Peer_Agent * agent = GNUNET_new (struct RIL_Peer_Agent);
1954 
1955  agent->envi = solver;
1956  agent->peer = *peer;
1957  agent->step_count = 0;
1958  agent->is_active = GNUNET_NO;
1959  agent->bw_in = RIL_MIN_BW;
1960  agent->bw_out = RIL_MIN_BW;
1961  agent->suggestion_issue = GNUNET_NO;
1962  agent->n = RIL_ACTION_TYPE_NUM;
1963  agent->m = 0;
1964  agent->W = (double **) GNUNET_malloc (sizeof (double *) * agent->n);
1965  agent->E = (double **) GNUNET_malloc (sizeof (double *) * agent->n);
1966  for (i = 0; i < agent->n; i++)
1967  {
1968  agent->W[i] = (double *) GNUNET_malloc (sizeof (double) * agent->m);
1969  agent->E[i] = (double *) GNUNET_malloc (sizeof (double) * agent->m);
1970  }
1971  agent_w_init(agent);
1972  agent->eligibility_reset = GNUNET_NO;
1973  agent->a_old = RIL_ACTION_INVALID;
1974  agent->s_old = GNUNET_malloc (sizeof (double) * agent->m);
1975  agent->address_inuse = NULL;
1976  agent->objective_old = 0;
1977  agent->nop_bonus = 0;
1978 
1979  return agent;
1980 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
unsigned int m
Number of rows of W / Number of state-vector features.
A handle for the reinforcement learning solver.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
int a_old
Last chosen action.
#define GNUNET_NO
Definition: gnunet_common.h:81
int suggestion_issue
Flag whether a suggestion has to be issued.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
struct GNUNET_PeerIdentity peer
Peer ID.
static void agent_w_init(struct RIL_Peer_Agent *agent)
Initializes the matrix W of parameter vectors theta with small random numbers.
double ** W
Experience matrix W.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.
double ** E
Eligibility traces.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
double * s_old
Last perceived state feature vector.
struct GAS_RIL_Handle * envi
Environment handle.
#define GNUNET_malloc(size)
Wrapper around malloc.
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
unsigned long long step_count
Number of performed time-steps.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_die()

static void agent_die ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Deallocate agent.

Parameters
solverthe solver handle
agentthe agent to retire

Definition at line 1989 of file plugin_ats_ril.c.

References RIL_Peer_Agent::E, GNUNET_free, GNUNET_free_non_null, RIL_Peer_Agent::n, RIL_Peer_Agent::s_old, and RIL_Peer_Agent::W.

Referenced by libgnunet_plugin_ats_ril_done().

1990 {
1991  int i;
1992 
1993  for (i = 0; i < agent->n; i++)
1994  {
1995  GNUNET_free_non_null(agent->W[i]);
1996  GNUNET_free_non_null(agent->E[i]);
1997  }
1998  GNUNET_free_non_null(agent->W);
1999  GNUNET_free_non_null(agent->E);
2000  GNUNET_free_non_null(agent->s_old);
2001  GNUNET_free(agent);
2002 }
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
double ** W
Experience matrix W.
double ** E
Eligibility traces.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ ril_get_agent()

static struct RIL_Peer_Agent* ril_get_agent ( struct GAS_RIL_Handle solver,
const struct GNUNET_PeerIdentity peer,
int  create 
)
static

Returns the agent for a peer.

Parameters
solverthe solver handle
peerthe identity of the peer
createwhether or not to create an agent, if none is allocated yet
Returns
the agent

Definition at line 2013 of file plugin_ats_ril.c.

References agent_init(), GAS_RIL_Handle::agents_head, GAS_RIL_Handle::agents_tail, GNUNET_CONTAINER_DLL_insert_tail, GNUNET_memcmp, RIL_Peer_Agent::next, and RIL_Peer_Agent::peer.

Referenced by GAS_ril_address_add(), GAS_ril_address_delete(), GAS_ril_get_preferred_address(), and GAS_ril_stop_get_preferred_address().

2014 {
2015  struct RIL_Peer_Agent *cur;
2016 
2017  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
2018  {
2019  if (0 == GNUNET_memcmp (peer, &cur->peer))
2020  {
2021  return cur;
2022  }
2023  }
2024 
2025  if (create)
2026  {
2027  cur = agent_init (solver, peer);
2029  return cur;
2030  }
2031  return NULL ;
2032 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
struct RIL_Peer_Agent * agents_tail
struct GNUNET_PeerIdentity peer
Peer ID.
static struct RIL_Peer_Agent * agent_init(void *s, const struct GNUNET_PeerIdentity *peer)
Initialize an agent without addresses and its knowledge base.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
#define GNUNET_memcmp(a, b)
Compare memory in a and b, where both must be of the same pointer type.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_network_is_active()

static int ril_network_is_active ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  network 
)
static

Determine whether at least the minimum bandwidth is set for the network.

Otherwise the network is considered inactive and not used. Addresses in an inactive network are ignored.

Parameters
solversolver handle
networkthe network type
Returns
whether or not the network is considered active

Definition at line 2043 of file plugin_ats_ril.c.

References RIL_Scope::bw_out_available, ril_get_network(), and RIL_MIN_BW.

Referenced by GAS_ril_address_add(), and GAS_ril_address_delete().

2044 {
2045  struct RIL_Scope *net;
2046 
2047  net = ril_get_network (solver, network);
2048  return net->bw_out_available >= RIL_MIN_BW;
2049 }
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
uint32_t bw_out_available
Total available outbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_cut_from_vector()

static void ril_cut_from_vector ( void **  old,
size_t  element_size,
unsigned int  hole_start,
unsigned int  hole_length,
unsigned int  old_length 
)
static

Cuts a slice out of a vector of elements.

This is used to decrease the size of the matrix storing the reward function approximation. It copies the memory, which is not cut, to the new vector, frees the memory of the old vector, and redirects the pointer to the new one.

Parameters
oldpointer to the pointer to the first element of the vector
element_sizebyte size of the vector elements
hole_startthe first element to cut out
hole_lengththe number of elements to cut out
old_lengththe length of the old vector

Definition at line 2063 of file plugin_ats_ril.c.

References GNUNET_assert, GNUNET_free, GNUNET_malloc, GNUNET_memcpy, and size.

Referenced by GAS_ril_address_delete().

2068 {
2069  char *tmpptr;
2070  char *oldptr = (char *) *old;
2071  size_t size;
2072  unsigned int bytes_before;
2073  unsigned int bytes_hole;
2074  unsigned int bytes_after;
2075 
2076  GNUNET_assert(old_length >= hole_length);
2077  GNUNET_assert(old_length >= (hole_start + hole_length));
2078 
2079  size = element_size * (old_length - hole_length);
2080 
2081  bytes_before = element_size * hole_start;
2082  bytes_hole = element_size * hole_length;
2083  bytes_after = element_size * (old_length - hole_start - hole_length);
2084 
2085  if (0 == size)
2086  {
2087  tmpptr = NULL;
2088  }
2089  else
2090  {
2091  tmpptr = GNUNET_malloc (size);
2092  GNUNET_memcpy (tmpptr, oldptr, bytes_before);
2093  GNUNET_memcpy (tmpptr + bytes_before, oldptr + (bytes_before + bytes_hole), bytes_after);
2094  }
2095  if (NULL != *old)
2096  {
2097  GNUNET_free(*old);
2098  }
2099  *old = (void *) tmpptr;
2100 }
static unsigned int element_size
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_memcpy(dst, src, n)
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ GAS_ril_address_change_preference()

static void GAS_ril_address_change_preference ( void *  solver,
const struct GNUNET_PeerIdentity peer,
enum GNUNET_ATS_PreferenceKind  kind,
double  pref_rel 
)
static

Change relative preference for quality in solver.

Parameters
solverthe solver handle
peerthe peer to change the preference for
kindthe kind to change the preference
pref_relthe normalized preference value for this kind over all clients

Definition at line 2116 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ATS_print_preference_type(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, GAS_RIL_Handle::parameters, ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2120 {
2122  "API_address_change_preference() Preference '%s' for peer '%s' changed to %.2f \n",
2123  GNUNET_ATS_print_preference_type (kind), GNUNET_i2s (peer), pref_rel);
2124 
2125  struct GAS_RIL_Handle *s = solver;
2126 
2129  ril_step (s);
2130 }
A handle for the reinforcement learning solver.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
double temperature
Softmax action-selection temperature.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_init
Initial exploration ratio value.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_add()

static void GAS_ril_address_add ( void *  solver,
struct ATS_Address address,
uint32_t  network 
)
static

Add a new address for a peer to the solver.

The address is already contained in the addresses hashmap!

Parameters
solverthe solver Handle
addressthe address to add
networknetwork type of this address

Definition at line 2143 of file plugin_ats_ril.c.

References ATS_Address::active, ATS_Address::addr, address, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::addresses_tail, RIL_Peer_Agent::E, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_array_grow, GNUNET_CONTAINER_DLL_insert_tail, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_new, GNUNET_NO, GNUNET_YES, LOG, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, ATS_Address::peer, ATS_Address::plugin, RIL_Learning_Parameters::rbf_divisor, ril_get_agent(), ril_get_network(), ril_network_is_active(), ril_step(), ril_try_unblock_agent(), RIL_Peer_Agent::s_old, ATS_Address::solver_information, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_init, RIL_Peer_Agent::W, and zero.

Referenced by libgnunet_plugin_ats_ril_init().

2146 {
2147  struct GAS_RIL_Handle *s = solver;
2148  struct RIL_Peer_Agent *agent;
2149  struct RIL_Address_Wrapped *address_wrapped;
2150  struct RIL_Scope *net;
2151  unsigned int m_new;
2152  unsigned int m_old;
2153  unsigned int n_new;
2154  unsigned int n_old;
2155  int i;
2156  unsigned int zero;
2157 
2159  "API_address_add()\n");
2160 
2161  net = ril_get_network (s, network);
2162  address->solver_information = net;
2163 
2164  if (!ril_network_is_active (s, network))
2165  {
2167  "API_address_add() Did not add %s address %s for peer '%s', network does not have enough bandwidth\n",
2168  address->plugin, address->addr, GNUNET_i2s (&address->peer));
2169  return;
2170  }
2171 
2174 
2175  agent = ril_get_agent (s, &address->peer, GNUNET_YES);
2176 
2177  //add address
2178  address_wrapped = GNUNET_new (struct RIL_Address_Wrapped);
2179  address_wrapped->address_naked = address;
2180  GNUNET_CONTAINER_DLL_insert_tail(agent->addresses_head, agent->addresses_tail, address_wrapped);
2181 
2182  //increase size of W
2183  m_new = agent->m + ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1));
2184  m_old = agent->m;
2185  n_new = agent->n + 1;
2186  n_old = agent->n;
2187 
2188  GNUNET_array_grow(agent->W, agent->n, n_new);
2189  agent->n = n_old;
2190  GNUNET_array_grow(agent->E, agent->n, n_new);
2191  for (i = 0; i < n_new; i++)
2192  {
2193  if (i < n_old)
2194  {
2195  agent->m = m_old;
2196  GNUNET_array_grow(agent->W[i], agent->m, m_new);
2197  agent->m = m_old;
2198  GNUNET_array_grow(agent->E[i], agent->m, m_new);
2199  }
2200  else
2201  {
2202  zero = 0;
2203  GNUNET_array_grow(agent->W[i], zero, m_new);
2204  zero = 0;
2205  GNUNET_array_grow(agent->E[i], zero, m_new);
2206  }
2207  }
2208 
2209  //increase size of old state vector
2210  agent->m = m_old;
2211  GNUNET_array_grow(agent->s_old, agent->m, m_new);
2212 
2213  ril_try_unblock_agent(s, agent, GNUNET_NO);
2214 
2215  ril_step (s);
2216 
2217  LOG(GNUNET_ERROR_TYPE_DEBUG, "API_address_add() Added %s %s address %s for peer '%s'\n",
2218  address->active ? "active" : "inactive", address->plugin, address->addr,
2219  GNUNET_i2s (&address->peer));
2220 }
static const struct GNUNET_CRYPTO_EcdsaPrivateKey zero
Public key of all zeros.
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
int active
Is this the active address for this peer?
unsigned int m
Number of rows of W / Number of state-vector features.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
#define GNUNET_NO
Definition: gnunet_common.h:81
char * plugin
Plugin name.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
struct ATS_Address * address_naked
The address.
double temperature
Softmax action-selection temperature.
#define GNUNET_array_grow(arr, size, tsize)
Grow a well-typed (!) array.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
double epsilon_init
Initial exploration ratio value.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
#define GNUNET_YES
Definition: gnunet_common.h:80
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
unsigned long long rbf_divisor
State space divisor.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
static char * address
GNS address for this phone.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Wrapper for addresses to store them in agent&#39;s linked list.
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_delete()

static void GAS_ril_address_delete ( void *  solver,
struct ATS_Address address 
)
static

Delete an address in the solver.

The address is not contained in the address hashmap anymore!

Parameters
solverthe solver handle
addressthe address to remove

Definition at line 2231 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, ATS_Address::active, ATS_Address::addr, RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), agent_address_get_wrapped(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::E, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_assert, GNUNET_CONTAINER_DLL_remove, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_free_non_null, GNUNET_i2s(), GNUNET_log, GNUNET_NO, GNUNET_YES, LOG, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, ATS_Address::peer, ATS_Address::plugin, RIL_Learning_Parameters::rbf_divisor, RIL_ACTION_INVALID, RIL_ACTION_TYPE_NUM, ril_cut_from_vector(), ril_get_agent(), ril_network_is_active(), ril_step(), RIL_Peer_Agent::s_old, ATS_Address::solver_information, RIL_Peer_Agent::suggestion_address, RIL_Peer_Agent::suggestion_issue, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_init, RIL_Scope::type, and RIL_Peer_Agent::W.

Referenced by libgnunet_plugin_ats_ril_init().

2233 {
2234  struct GAS_RIL_Handle *s = solver;
2235  struct RIL_Peer_Agent *agent;
2236  struct RIL_Address_Wrapped *address_wrapped;
2237  int address_index;
2238  unsigned int m_new;
2239  unsigned int n_new;
2240  int i;
2241  struct RIL_Scope *net;
2242 
2244  "API_address_delete() Delete %s %s address %s for peer '%s'\n",
2245  address->active ? "active" : "inactive",
2246  address->plugin,
2247  address->addr,
2248  GNUNET_i2s (&address->peer));
2249 
2250  agent = ril_get_agent (s, &address->peer, GNUNET_NO);
2251  if (NULL == agent)
2252  {
2253  net = address->solver_information;
2256  "No agent allocated for peer yet, since address was in inactive network\n");
2257  return;
2258  }
2259 
2262 
2263  address_index = agent_address_get_index (agent, address);
2264  address_wrapped = agent_address_get_wrapped (agent, address);
2265 
2266  if (NULL == address_wrapped)
2267  {
2268  net = address->solver_information;
2270  "Address not considered by agent, address was in inactive network\n");
2271  return;
2272  }
2274  agent->addresses_tail,
2275  address_wrapped);
2276  GNUNET_free (address_wrapped);
2277 
2278  //decrease W
2279  m_new = agent->m - ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1));
2280  n_new = agent->n - 1;
2281 
2282  for (i = 0; i < agent->n; i++)
2283  {
2284  ril_cut_from_vector ((void **) &agent->W[i], sizeof(double),
2285  address_index * ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)),
2286  ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), agent->m);
2287  ril_cut_from_vector ((void **) &agent->E[i], sizeof(double),
2288  address_index * ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)),
2289  ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), agent->m);
2290  }
2291  GNUNET_free_non_null(agent->W[RIL_ACTION_TYPE_NUM + address_index]);
2292  GNUNET_free_non_null(agent->E[RIL_ACTION_TYPE_NUM + address_index]);
2293  ril_cut_from_vector ((void **) &agent->W, sizeof(double *), RIL_ACTION_TYPE_NUM + address_index,
2294  1, agent->n);
2295  ril_cut_from_vector ((void **) &agent->E, sizeof(double *), RIL_ACTION_TYPE_NUM + address_index,
2296  1, agent->n);
2297  //correct last action
2298  if (agent->a_old > (RIL_ACTION_TYPE_NUM + address_index))
2299  {
2300  agent->a_old -= 1;
2301  }
2302  else if (agent->a_old == (RIL_ACTION_TYPE_NUM + address_index))
2303  {
2304  agent->a_old = RIL_ACTION_INVALID;
2305  }
2306  //decrease old state vector
2307  ril_cut_from_vector ((void **) &agent->s_old, sizeof(double),
2308  address_index * ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)),
2309  ((s->parameters.rbf_divisor+1) * (s->parameters.rbf_divisor+1)), agent->m);
2310  agent->m = m_new;
2311  agent->n = n_new;
2312 
2313  if (agent->address_inuse == address)
2314  {
2315  if (NULL != agent->addresses_head) //if peer has an address left, use it
2316  {
2318  "Active address died, suggesting alternative!\n");
2320  agent,
2321  agent->addresses_head->address_naked,
2322  agent->bw_in,
2323  agent->bw_out,
2324  GNUNET_YES);
2325  }
2326  else
2327  {
2329  "Active address died, suggesting disconnect!\n");
2330  envi_set_active_suggestion (s, agent, NULL, 0, 0, GNUNET_NO);
2331  }
2332  }
2333  ril_step (solver);
2334  if (agent->suggestion_address == address)
2335  {
2336  agent->suggestion_issue = GNUNET_NO;
2337  agent->suggestion_address = NULL;
2338  }
2339  GNUNET_assert (agent->address_inuse != address);
2340 }
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int active
Is this the active address for this peer?
unsigned int m
Number of rows of W / Number of state-vector features.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
int a_old
Last chosen action.
static void ril_cut_from_vector(void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
Cuts a slice out of a vector of elements.
#define GNUNET_NO
Definition: gnunet_common.h:81
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
char * plugin
Plugin name.
int suggestion_issue
Flag whether a suggestion has to be issued.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
double temperature
Softmax action-selection temperature.
enum GNUNET_NetworkType type
ATS network type.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
static struct RIL_Address_Wrapped * agent_address_get_wrapped(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Gets the wrapped address from the agent&#39;s list.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_log(kind,...)
double * s_old
Last perceived state feature vector.
#define GNUNET_YES
Definition: gnunet_common.h:80
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
unsigned long long rbf_divisor
State space divisor.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
struct ATS_Address * suggestion_address
The address which has to be issued.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Wrapper for addresses to store them in agent&#39;s linked list.
double temperature_init
Initial softmax action-selection temperature.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_property_changed()

static void GAS_ril_address_property_changed ( void *  solver,
struct ATS_Address address 
)
static

Update the properties of an address in the solver.

Parameters
solversolver handle
addressthe address

Definition at line 2350 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, GAS_RIL_Handle::parameters, ATS_Address::peer, ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2352 {
2353  struct GAS_RIL_Handle *s = solver;
2354 
2356  "Properties for peer '%s' address changed\n",
2357  GNUNET_i2s (&address->peer));
2360  ril_step (s);
2361 }
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
A handle for the reinforcement learning solver.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_init
Initial exploration ratio value.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_preference_feedback()

static void GAS_ril_address_preference_feedback ( void *  solver,
struct GNUNET_SERVICE_Client application,
const struct GNUNET_PeerIdentity peer,
const struct GNUNET_TIME_Relative  scope,
enum GNUNET_ATS_PreferenceKind  kind,
double  score 
)
static

Give feedback about the current assignment.

Parameters
solverthe solver handle
applicationthe application
peerthe peer to change the preference for
scopethe time interval for this feedback: [now - scope .. now]
kindthe kind to change the preference
scorethe score

Definition at line 2375 of file plugin_ats_ril.c.

References GNUNET_ATS_print_preference_type(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, and GNUNET_TIME_Relative::rel_value_us.

Referenced by libgnunet_plugin_ats_ril_init().

2381 {
2383  "API_address_preference_feedback() Peer '%s' got a feedback of %+.3f from application %s for "
2384  "preference %s for %d seconds\n",
2385  GNUNET_i2s (peer),
2386  "UNKNOWN",
2388  scope.rel_value_us / 1000000);
2389 }
uint64_t rel_value_us
The actual value.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
#define LOG(kind,...)
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_bulk_start()

static void GAS_ril_bulk_start ( void *  solver)
static

Start a bulk operation.

Parameters
solverthe solver

Definition at line 2398 of file plugin_ats_ril.c.

References GAS_RIL_Handle::bulk_lock, GNUNET_ERROR_TYPE_DEBUG, and LOG.

Referenced by libgnunet_plugin_ats_ril_init().

2399 {
2400  struct GAS_RIL_Handle *s = solver;
2401 
2403  "API_bulk_start() lock: %d\n", s->bulk_lock+1);
2404 
2405  s->bulk_lock++;
2406 }
A handle for the reinforcement learning solver.
int bulk_lock
Lock for bulk operations.
#define LOG(kind,...)
Here is the caller graph for this function:

◆ GAS_ril_bulk_stop()

static void GAS_ril_bulk_stop ( void *  solver)
static

Bulk operation done.

Parameters
solverthe solver handle

Definition at line 2415 of file plugin_ats_ril.c.

References GAS_RIL_Handle::bulk_changes, GAS_RIL_Handle::bulk_lock, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, LOG, and ril_step().

Referenced by libgnunet_plugin_ats_ril_init().

2416 {
2417  struct GAS_RIL_Handle *s = solver;
2418 
2420  "API_bulk_stop() lock: %d\n",
2421  s->bulk_lock - 1);
2422 
2423  if (s->bulk_lock < 1)
2424  {
2425  GNUNET_break(0);
2426  return;
2427  }
2428  s->bulk_lock--;
2429 
2430  if (0 < s->bulk_changes)
2431  {
2432  ril_step (solver);
2433  s->bulk_changes = 0;
2434  }
2435 }
int bulk_changes
Number of changes during a lock.
A handle for the reinforcement learning solver.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
int bulk_lock
Lock for bulk operations.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_get_preferred_address()

static void GAS_ril_get_preferred_address ( void *  solver,
const struct GNUNET_PeerIdentity peer 
)
static

Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth changed callback.

The solver must only notify about changes for peers with pending address requests!

Parameters
solverthe solver handle
peerthe identity of the peer

Definition at line 2449 of file plugin_ats_ril.c.

References ATS_Address::addr, RIL_Peer_Agent::address_inuse, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_YES, RIL_Peer_Agent::is_active, LOG, GAS_RIL_Handle::parameters, ATS_Address::plugin, ril_get_agent(), ril_try_unblock_agent(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2451 {
2452  struct GAS_RIL_Handle *s = solver;
2453  struct RIL_Peer_Agent *agent;
2454 
2455  LOG(GNUNET_ERROR_TYPE_DEBUG, "API_get_preferred_address()\n");
2456 
2457  agent = ril_get_agent (s, peer, GNUNET_YES);
2458 
2459  agent->is_active = GNUNET_YES;
2460  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in, agent->bw_out, GNUNET_YES);
2461 
2462  ril_try_unblock_agent(solver, agent, GNUNET_YES);
2463 
2464  if (agent->address_inuse)
2465  {
2467  "API_get_preferred_address() Activated agent for peer '%s' with %s address %s\n",
2468  GNUNET_i2s (peer), agent->address_inuse->plugin, agent->address_inuse->addr);
2469  }
2470  else
2471  {
2473  "API_get_preferred_address() Activated agent for peer '%s', but no address available\n",
2474  GNUNET_i2s (peer));
2477  }
2478  if (NULL != agent->address_inuse)
2479  s->env->bandwidth_changed_cb (s->env->cls,
2480  agent->address_inuse);
2481 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
char * plugin
Plugin name.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:80
#define LOG(kind,...)
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_stop_get_preferred_address()

static void GAS_ril_stop_get_preferred_address ( void *  solver,
const struct GNUNET_PeerIdentity peer 
)
static

Tell solver stop notifying ATS about changes for this peers.

The solver must only notify about changes for peers with pending address requests!

Parameters
solverthe solver handle
peerthe peer

Definition at line 2494 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::is_active, LOG, GAS_RIL_Handle::parameters, ril_get_agent(), ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2496 {
2497  struct GAS_RIL_Handle *s = solver;
2498  struct RIL_Peer_Agent *agent;
2499 
2501  "API_stop_get_preferred_address()");
2502 
2503  agent = ril_get_agent (s, peer, GNUNET_NO);
2504 
2505  if (NULL == agent)
2506  {
2507  GNUNET_break(0);
2508  return;
2509  }
2510  if (GNUNET_NO == agent->is_active)
2511  {
2512  GNUNET_break(0);
2513  return;
2514  }
2515 
2518 
2519  agent->is_active = GNUNET_NO;
2520 
2521  envi_set_active_suggestion (s, agent, agent->address_inuse, agent->bw_in, agent->bw_out,
2522  GNUNET_YES);
2523 
2524  ril_step (s);
2525 
2527  "API_stop_get_preferred_address() Paused agent for peer '%s'\n",
2528  GNUNET_i2s (peer));
2529 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
A handle for the reinforcement learning solver.
#define GNUNET_NO
Definition: gnunet_common.h:81
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
#define GNUNET_YES
Definition: gnunet_common.h:80
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ libgnunet_plugin_ats_ril_init()

void* libgnunet_plugin_ats_ril_init ( void *  cls)

Entry point for the plugin.

Parameters
clspointer to the 'struct GNUNET_ATS_PluginEnvironment'

Definition at line 2538 of file plugin_ats_ril.c.

References _, RIL_Learning_Parameters::algorithm, RIL_Learning_Parameters::alpha, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, RIL_Learning_Parameters::beta, RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, GNUNET_ATS_PluginEnvironment::cfg, GNUNET_ATS_SolverFunctions::cls, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::done, RIL_Learning_Parameters::eligibility_trace_mode, env, GAS_RIL_Handle::env, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_decay, RIL_Learning_Parameters::epsilon_init, RIL_Learning_Parameters::gamma, GAS_ril_address_add(), GAS_ril_address_change_preference(), GAS_ril_address_delete(), GAS_ril_address_preference_feedback(), GAS_ril_address_property_changed(), GAS_ril_bulk_start(), GAS_ril_bulk_stop(), GAS_ril_get_preferred_address(), GAS_ril_stop_get_preferred_address(), GNUNET_ATS_PluginEnvironment::get_preferences, GNUNET_assert, GNUNET_CONFIGURATION_get_value_float(), GNUNET_CONFIGURATION_get_value_number(), GNUNET_CONFIGURATION_get_value_string(), GNUNET_CONFIGURATION_get_value_time(), GNUNET_CONFIGURATION_get_value_yesno(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_free, GNUNET_malloc, GNUNET_new, GNUNET_NO, GNUNET_NT_to_string(), GNUNET_OK, GNUNET_STRINGS_utf8_toupper(), GNUNET_SYSERR, GNUNET_YES, GNUNET_ATS_PluginEnvironment::in_quota, RIL_Learning_Parameters::lambda, LOG, GNUNET_ATS_PluginEnvironment::network_count, GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, GNUNET_ATS_PluginEnvironment::out_quota, GAS_RIL_Handle::parameters, RIL_Learning_Parameters::rbf_divisor, RIL_ALGO_Q, RIL_ALGO_SARSA, RIL_DEFAULT_ALGORITHM, RIL_DEFAULT_DISCOUNT_BETA, RIL_DEFAULT_DISCOUNT_GAMMA, RIL_DEFAULT_EXPLORE_DECAY, RIL_DEFAULT_EXPLORE_RATIO, RIL_DEFAULT_GRADIENT_STEP_SIZE, RIL_DEFAULT_RBF_DIVISOR, RIL_DEFAULT_SELECT, RIL_DEFAULT_STEP_TIME_MAX, RIL_DEFAULT_STEP_TIME_MIN, RIL_DEFAULT_TEMPERATURE, RIL_DEFAULT_TEMPERATURE_DECAY, RIL_DEFAULT_TRACE_DECAY, RIL_DEFAULT_WELFARE, RIL_E_ACCUMULATE, RIL_E_REPLACE, RIL_SELECT_EGREEDY, RIL_SELECT_SOFTMAX, RIL_WELFARE_EGALITARIAN, RIL_WELFARE_NASH, GNUNET_ATS_SolverFunctions::s_add, GNUNET_ATS_SolverFunctions::s_address_update_property, GNUNET_ATS_SolverFunctions::s_bulk_start, GNUNET_ATS_SolverFunctions::s_bulk_stop, GNUNET_ATS_SolverFunctions::s_del, GNUNET_ATS_SolverFunctions::s_feedback, GNUNET_ATS_SolverFunctions::s_get, GNUNET_ATS_SolverFunctions::s_get_stop, GNUNET_ATS_SolverFunctions::s_pref, RIL_Learning_Parameters::select, sf, GAS_RIL_Handle::simulate, RIL_Learning_Parameters::social_welfare, GNUNET_ATS_PluginEnvironment::stats, GAS_RIL_Handle::step_count, RIL_Learning_Parameters::step_time_max, RIL_Learning_Parameters::step_time_min, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_decay, RIL_Learning_Parameters::temperature_init, and RIL_Scope::type.

2539 {
2540  static struct GNUNET_ATS_SolverFunctions sf;
2542  struct GAS_RIL_Handle *solver = GNUNET_new (struct GAS_RIL_Handle);
2543  struct RIL_Scope * cur;
2544  int c;
2545  char *string;
2546  float f_tmp;
2547 
2549  "API_init() Initializing RIL solver\n");
2550 
2551  GNUNET_assert (NULL != env);
2552  GNUNET_assert (NULL != env->cfg);
2553  GNUNET_assert (NULL != env->stats);
2554  GNUNET_assert (NULL != env->bandwidth_changed_cb);
2555  GNUNET_assert (NULL != env->get_preferences);
2556 
2557  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number(env->cfg, "ats", "RIL_RBF_DIVISOR", &solver->parameters.rbf_divisor))
2558  {
2560  }
2561 
2562  if (GNUNET_OK
2563  != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats", "RIL_STEP_TIME_MIN",
2564  &solver->parameters.step_time_min))
2565  {
2567  }
2568 
2569  if (GNUNET_OK
2570  != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats", "RIL_STEP_TIME_MAX",
2571  &solver->parameters.step_time_max))
2572  {
2574  }
2575 
2576  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_ALGORITHM", &string))
2577  {
2578  GNUNET_STRINGS_utf8_toupper (string, string);
2579  if (0 == strcmp (string, "SARSA"))
2580  {
2582  }
2583  if (0 == strcmp (string, "Q-LEARNING"))
2584  {
2585  solver->parameters.algorithm = RIL_ALGO_Q;
2586  }
2587 
2588  GNUNET_free (string);
2589  }
2590  else
2591  {
2593  }
2594 
2595  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_SELECT", &string))
2596  {
2597  solver->parameters.select = !strcmp (string, "EGREEDY") ? RIL_SELECT_EGREEDY : RIL_SELECT_SOFTMAX;
2598  GNUNET_free (string);
2599  }
2600  else
2601  {
2603  }
2604 
2605 
2608  "RIL_DISCOUNT_BETA", &f_tmp))
2609  {
2610  if (f_tmp < 0.0)
2611  {
2612  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2613  "RIL_DISCOUNT_BETA", f_tmp);
2614  }
2615  else
2616  {
2617  solver->parameters.beta = f_tmp;
2618  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2619  "RIL_DISCOUNT_BETA", f_tmp);
2620  }
2621  }
2622 
2625  "RIL_DISCOUNT_GAMMA", &f_tmp))
2626  {
2627  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2628  {
2629  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2630  "RIL_DISCOUNT_GAMMA", f_tmp);
2631  }
2632  else
2633  {
2634  solver->parameters.gamma = f_tmp;
2635  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2636  "RIL_DISCOUNT_GAMMA", f_tmp);
2637  }
2638  }
2639 
2642  "RIL_GRADIENT_STEP_SIZE", &f_tmp))
2643  {
2644  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2645  {
2646  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2647  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2648  }
2649  else
2650  {
2651  solver->parameters.alpha = f_tmp;
2652  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2653  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2654  }
2655  }
2656 
2659  "RIL_TRACE_DECAY", &f_tmp))
2660  {
2661  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2662  {
2663  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2664  "RIL_TRACE_DECAY", f_tmp);
2665  }
2666  else
2667  {
2668  solver->parameters.lambda = f_tmp;
2669  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2670  "RIL_TRACE_DECAY", f_tmp);
2671  }
2672  }
2673 
2676  "RIL_EXPLORE_RATIO", &f_tmp))
2677  {
2678  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2679  {
2680  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2681  "RIL_EXPLORE_RATIO", f_tmp);
2682  }
2683  else
2684  {
2685  solver->parameters.epsilon_init = f_tmp;
2686  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2687  "RIL_EXPLORE_RATIO", f_tmp);
2688  }
2689  }
2690 
2693  "RIL_EXPLORE_DECAY", &f_tmp))
2694  {
2695  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2696  {
2697  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2698  "RIL_EXPLORE_DECAY", f_tmp);
2699  }
2700  else
2701  {
2702  solver->parameters.epsilon_decay = f_tmp;
2703  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2704  "RIL_EXPLORE_DECAY", f_tmp);
2705  }
2706  }
2707 
2710  "RIL_TEMPERATURE", &f_tmp))
2711  {
2712  if (f_tmp <= 0.0)
2713  {
2714  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2715  "RIL_TEMPERATURE", f_tmp);
2716  }
2717  else
2718  {
2719  solver->parameters.temperature_init = f_tmp;
2720  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2721  "RIL_TEMPERATURE", f_tmp);
2722  }
2723  }
2724 
2727  "RIL_TEMPERATURE_DECAY", &f_tmp))
2728  {
2729  if ((f_tmp <= 0.0) || solver->parameters.temperature_decay > 1)
2730  {
2731  LOG (GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2732  "RIL_TEMPERATURE_DECAY", f_tmp);
2733  }
2734  else
2735  {
2736  solver->parameters.temperature_decay = f_tmp;
2737  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2738  "RIL_TEMPERATURE_DECAY", f_tmp);
2739  }
2740  }
2741 
2742  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (env->cfg, "ats", "RIL_SIMULATE", &solver->simulate))
2743  {
2744  solver->simulate = GNUNET_NO;
2745  }
2746 
2747  if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(env->cfg, "ats", "RIL_REPLACE_TRACES"))
2748  {
2750  }
2751  else
2752  {
2754  }
2755 
2756  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (env->cfg, "ats", "RIL_SOCIAL_WELFARE", &string))
2757  {
2758  solver->parameters.social_welfare = !strcmp (string, "NASH") ? RIL_WELFARE_NASH : RIL_WELFARE_EGALITARIAN;
2759  GNUNET_free (string);
2760  }
2761  else
2762  {
2764  }
2765 
2766  solver->env = env;
2767  sf.cls = solver;
2777 
2778  solver->networks_count = env->network_count;
2779  solver->network_entries = GNUNET_malloc (env->network_count * sizeof (struct RIL_Scope));
2780  solver->step_count = 0;
2781  solver->done = GNUNET_NO;
2782 
2783  for (c = 0; c < env->network_count; c++)
2784  {
2785  cur = &solver->network_entries[c];
2786  cur->type = c;
2787  cur->bw_in_available = env->in_quota[c];
2788  cur->bw_out_available = env->out_quota[c];
2790  "init() Quotas for %s network: IN %llu - OUT %llu\n",
2791  GNUNET_NT_to_string(cur->type),
2792  cur->bw_in_available/1024,
2793  cur->bw_out_available/1024);
2794  }
2795 
2796  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() Parameters:\n");
2797  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() Algorithm = %s, alpha = %f, beta = %f, lambda = %f\n",
2798  solver->parameters.algorithm ? "Q" : "SARSA",
2799  solver->parameters.alpha,
2800  solver->parameters.beta,
2801  solver->parameters.lambda);
2802  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() exploration_ratio = %f, temperature = %f, ActionSelection = %s\n",
2803  solver->parameters.epsilon,
2804  solver->parameters.temperature,
2805  solver->parameters.select ? "EGREEDY" : "SOFTMAX");
2806  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() RBF_DIVISOR = %llu\n",
2807  solver->parameters.rbf_divisor);
2808 
2809  return &sf;
2810 }
GAS_solver_stop_get_preferred_address s_get_stop
Tell solver stop notifying ATS about changes for this peers.
static void GAS_ril_address_change_preference(void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
Change relative preference for quality in solver.
static void GAS_ril_address_delete(void *solver, struct ATS_Address *address)
Delete an address in the solver.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
int GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
GAS_solver_address_add s_add
Add a new address for a peer to the solver.
int GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
const char * GNUNET_NT_to_string(enum GNUNET_NetworkType net)
Convert a enum GNUNET_NetworkType to a string.
Definition: nt.c:43
#define RIL_DEFAULT_SELECT
double lambda
Trace-decay factor for eligibility traces.
struct GNUNET_STATISTICS_Handle * stats
Statistics handle to be used by the solver.
#define RIL_DEFAULT_DISCOUNT_GAMMA
#define RIL_DEFAULT_EXPLORE_DECAY
#define RIL_DEFAULT_RBF_DIVISOR
static void GAS_ril_address_add(void *solver, struct ATS_Address *address, uint32_t network)
Add a new address for a peer to the solver.
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
static void GAS_ril_bulk_stop(void *solver)
Bulk operation done.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void GAS_ril_stop_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver stop notifying ATS about changes for this peers.
A handle for the reinforcement learning solver.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define RIL_DEFAULT_GRADIENT_STEP_SIZE
GAS_solver_address_feedback_preference s_feedback
Give feedback about the current assignment.
int GNUNET_CONFIGURATION_get_value_float(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, float *number)
Get a configuration value that should be a floating point number.
GAS_solver_address_property_changed s_address_update_property
Update the properties of an address in the solver.
double temperature_decay
Decay factor of the temperature value.
#define RIL_DEFAULT_WELFARE
#define GNUNET_NO
Definition: gnunet_common.h:81
struct RIL_Scope * network_entries
Array of networks with global assignment state.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
#define GNUNET_new(type)
Allocate a struct or union of the given type.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void GAS_ril_address_property_changed(void *solver, struct ATS_Address *address)
Update the properties of an address in the solver.
unsigned long long step_count
Number of performed steps.
double temperature
Softmax action-selection temperature.
#define _(String)
GNU gettext support macro.
Definition: platform.h:208
#define RIL_DEFAULT_TRACE_DECAY
unsigned long long simulate
Simulate steps, i.e.
void GNUNET_STRINGS_utf8_toupper(const char *input, char *output)
Convert the utf-8 input string to upper case.
Definition: strings.c:580
static void GAS_ril_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
#define RIL_DEFAULT_ALGORITHM
enum RIL_Select select
Action selection strategy;.
enum GNUNET_NetworkType type
ATS network type.
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.
double beta
Learning discount variable in the TD-update for semi-MDPs.
GAS_solver_address_delete s_del
Delete an address in the solver.
#define RIL_DEFAULT_TEMPERATURE_DECAY
enum RIL_Algorithm algorithm
The TD-algorithm to use.
#define RIL_DEFAULT_STEP_TIME_MAX
struct RIL_Learning_Parameters parameters
Learning parameters.
GAS_solver_address_change_preference s_pref
Change relative preference for quality in solver.
#define RIL_DEFAULT_TEMPERATURE
int GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
#define RIL_DEFAULT_EXPLORE_RATIO
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
unsigned int networks_count
Networks count.
double epsilon_decay
Decay factor of the explore ratio.
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
void * cls
Closure to pass to all solver functions in this struct.
#define RIL_DEFAULT_STEP_TIME_MIN
double epsilon_init
Initial exploration ratio value.
GAS_solver_get_preferred_address s_get
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
unsigned long long in_quota[6]
Array of configured inbound quotas Order according to networks in network array.
#define RIL_DEFAULT_DISCOUNT_BETA
uint32_t bw_out_available
Total available outbound bandwidth.
GAS_solver_bulk_start s_bulk_start
Start a bulk operation.
int done
Shutdown.
static void GAS_ril_address_preference_feedback(void *solver, struct GNUNET_SERVICE_Client *application, const struct GNUNET_PeerIdentity *peer, const struct GNUNET_TIME_Relative scope, enum GNUNET_ATS_PreferenceKind kind, double score)
Give feedback about the current assignment.
static void GAS_ril_bulk_start(void *solver)
Start a bulk operation.
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration handle to be used by the solver.
unsigned long long out_quota[6]
Array of configured outbound quotas Order according to networks in network array. ...
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:80
static struct GNUNET_ATS_SolverFunctions * sf
Solver handle.
The ATS plugin will pass a pointer to a struct of this type as to the initialization function of the ...
Defi