GNUnet  0.10.x
Data Structures | Macros | Enumerations | Functions
plugin_ats_ril.c File Reference

ATS reinforcement learning solver. More...

#include "platform.h"
#include <float.h>
#include <math.h>
#include "gnunet_ats_plugin.h"
#include "gnunet-service-ats_addresses.h"
Include dependency graph for plugin_ats_ril.c:

Go to the source code of this file.

Data Structures

struct  RIL_Learning_Parameters
 Global learning parameters. More...
 
struct  RIL_Address_Wrapped
 Wrapper for addresses to store them in agent's linked list. More...
 
struct  RIL_Peer_Agent
 
struct  RIL_Scope
 
struct  GAS_RIL_Handle
 A handle for the reinforcement learning solver. More...
 

Macros

#define LOG(kind, ...)   GNUNET_log_from(kind, "ats-ril", __VA_ARGS__)
 
#define RIL_MIN_BW   (5 * ntohl(GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__))
 
#define RIL_MAX_BW   GNUNET_ATS_MaxBandwidth
 
#define RIL_ACTION_INVALID   -1
 
#define RIL_INTERVAL_EXPONENT   10
 
#define RIL_UTILITY_DELAY_MAX   1000
 
#define RIL_DEFAULT_STEP_TIME_MIN   GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 200)
 
#define RIL_DEFAULT_STEP_TIME_MAX   GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 2000)
 
#define RIL_DEFAULT_ALGORITHM   RIL_ALGO_SARSA
 
#define RIL_DEFAULT_SELECT   RIL_SELECT_SOFTMAX
 
#define RIL_DEFAULT_WELFARE   RIL_WELFARE_NASH
 
#define RIL_DEFAULT_DISCOUNT_BETA   0.6
 
#define RIL_DEFAULT_DISCOUNT_GAMMA   0.5
 
#define RIL_DEFAULT_GRADIENT_STEP_SIZE   0.01
 
#define RIL_DEFAULT_TRACE_DECAY   0.5
 
#define RIL_DEFAULT_EXPLORE_RATIO   1
 
#define RIL_DEFAULT_EXPLORE_DECAY   0.95
 
#define RIL_DEFAULT_RBF_DIVISOR   50
 
#define RIL_DEFAULT_TEMPERATURE   0.1
 
#define RIL_DEFAULT_TEMPERATURE_DECAY   1
 
#define RIL_INC_DEC_STEP_SIZE   1
 
#define RIL_NOP_DECAY   0.5
 

Enumerations

enum  RIL_Action_Type {
  RIL_ACTION_NOTHING = 0, RIL_ACTION_BW_IN_DBL = -2, RIL_ACTION_BW_IN_HLV = -3, RIL_ACTION_BW_IN_INC = 1,
  RIL_ACTION_BW_IN_DEC = 2, RIL_ACTION_BW_OUT_DBL = -4, RIL_ACTION_BW_OUT_HLV = -5, RIL_ACTION_BW_OUT_INC = 3,
  RIL_ACTION_BW_OUT_DEC = 4, RIL_ACTION_TYPE_NUM = 5
}
 ATS reinforcement learning solver. More...
 
enum  RIL_Algorithm { RIL_ALGO_SARSA = 0, RIL_ALGO_Q = 1 }
 
enum  RIL_Select { RIL_SELECT_SOFTMAX = 0, RIL_SELECT_EGREEDY = 1 }
 
enum  RIL_Welfare { RIL_WELFARE_NASH, RIL_WELFARE_EGALITARIAN }
 
enum  RIL_E_Modification { RIL_E_DECAY, RIL_E_ZERO, RIL_E_ACCUMULATE, RIL_E_REPLACE }
 

Functions

static double agent_q (struct RIL_Peer_Agent *agent, const double *state, int action)
 Estimate the current action-value for state s and action a. More...
 
static int agent_address_get_index (struct RIL_Peer_Agent *agent, struct ATS_Address *address)
 Get the index of the address in the agent's list. More...
 
static struct RIL_Address_Wrappedagent_address_get_wrapped (struct RIL_Peer_Agent *agent, struct ATS_Address *address)
 Gets the wrapped address from the agent's list. More...
 
static int agent_action_is_possible (struct RIL_Peer_Agent *agent, int action)
 
static int agent_get_action_max (struct RIL_Peer_Agent *agent, double *state)
 Gets the action, with the maximal estimated Q-value (i.e. More...
 
static int agent_get_action_random (struct RIL_Peer_Agent *agent)
 Chooses a random action from the set of possible ones. More...
 
static void agent_update (struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
 Updates the weights (i.e. More...
 
static void agent_modify_eligibility (struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
 Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each component as in accumulating eligibility traces RIL_E_REPLACE - resets each component to feature as in replacing traces RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing. More...
 
static void ril_inform (struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
 Informs the environment about the status of the solver. More...
 
static unsigned long long ril_get_max_bw (struct RIL_Scope *net)
 Calculates the maximum bandwidth an agent can assign in a network scope. More...
 
static void envi_set_active_suggestion (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
 Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify ATS of its new decision. More...
 
static double * envi_get_state (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Allocates a state vector and fills it with the features present. More...
 
static double agent_get_utility (struct RIL_Peer_Agent *agent)
 Returns the utility value of the connection an agent manages. More...
 
static double ril_network_get_social_welfare (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
 Calculates the social welfare within a network scope according to what social welfare measure is set in the configuration. More...
 
static double envi_get_penalty (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 
static double envi_get_reward (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Gets the reward for the last performed step, which is calculated in equal parts from the local (the peer specific) and the global (for all peers identical) reward. More...
 
static void envi_action_bw_double (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Doubles the bandwidth for the active address. More...
 
static void envi_action_bw_halven (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Cuts the bandwidth for the active address in half. More...
 
static void envi_action_bw_inc (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Increases the bandwidth by 5 times the minimum bandwidth for the active address. More...
 
static void envi_action_bw_dec (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Decreases the bandwidth by 5 times the minimum bandwidth for the active address. More...
 
static void envi_action_address_switch (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
 Switches to the address given by its index. More...
 
static void envi_do_action (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
 Puts the action into effect by calling the according function. More...
 
static int agent_select_egreedy (struct RIL_Peer_Agent *agent, double *state)
 Selects the next action using the e-greedy strategy. More...
 
static int agent_select_softmax (struct RIL_Peer_Agent *agent, double *state)
 Selects the next action with a probability corresponding to its value. More...
 
static int agent_select_action (struct RIL_Peer_Agent *agent, double *state)
 Select the next action of an agent either according to the e-greedy strategy or the softmax strategy. More...
 
static void agent_step (struct RIL_Peer_Agent *agent)
 Performs one step of the Markov Decision Process. More...
 
static void ril_step (struct GAS_RIL_Handle *solver)
 Prototype of the ril_step() procedure. More...
 
static void ril_step_scheduler_task (void *cls)
 Task for the scheduler, which performs one step and lets the solver know that no further step is scheduled. More...
 
static double ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver)
 Determines how much of the available bandwidth is assigned. More...
 
static struct RIL_Scoperil_get_network (struct GAS_RIL_Handle *s, uint32_t type)
 Lookup network struct by type. More...
 
static int ril_network_is_not_full (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
 Determines whether more connections are allocated in a network scope, than they would theoretically fit. More...
 
static void ril_try_unblock_agent (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
 Unblocks an agent for which a connection request is there, that could not be satisfied. More...
 
static void ril_calculate_discount (struct GAS_RIL_Handle *solver)
 Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step. More...
 
static int ril_network_count_active_agents (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
 Count the number of active agents/connections in a network scope. More...
 
static unsigned long long ril_network_get_assigned (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
 Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the outbound direction. More...
 
static unsigned long long ril_network_get_utilized (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
 Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound or in the outbound direction. More...
 
static void ril_networks_update_state (struct GAS_RIL_Handle *solver)
 Retrieves the state of the network scope, so that its attributes are up-to- date. More...
 
static void ril_step_schedule_next (struct GAS_RIL_Handle *solver)
 Schedules the next global step in an adaptive way. More...
 
static void agent_w_init (struct RIL_Peer_Agent *agent)
 Initializes the matrix W of parameter vectors theta with small random numbers. More...
 
static struct RIL_Peer_Agentagent_init (void *s, const struct GNUNET_PeerIdentity *peer)
 Initialize an agent without addresses and its knowledge base. More...
 
static void agent_die (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Deallocate agent. More...
 
static struct RIL_Peer_Agentril_get_agent (struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
 Returns the agent for a peer. More...
 
static int ril_network_is_active (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
 Determine whether at least the minimum bandwidth is set for the network. More...
 
static void ril_cut_from_vector (void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
 Cuts a slice out of a vector of elements. More...
 
static void GAS_ril_address_change_preference (void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
 Change relative preference for quality in solver. More...
 
static void GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network)
 Add a new address for a peer to the solver. More...
 
static void GAS_ril_address_delete (void *solver, struct ATS_Address *address)
 Delete an address in the solver. More...
 
static void GAS_ril_address_property_changed (void *solver, struct ATS_Address *address)
 Update the properties of an address in the solver. More...
 
static void GAS_ril_address_preference_feedback (void *solver, struct GNUNET_SERVICE_Client *application, const struct GNUNET_PeerIdentity *peer, const struct GNUNET_TIME_Relative scope, enum GNUNET_ATS_PreferenceKind kind, double score)
 Give feedback about the current assignment. More...
 
static void GAS_ril_bulk_start (void *solver)
 Start a bulk operation. More...
 
static void GAS_ril_bulk_stop (void *solver)
 Bulk operation done. More...
 
static void GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer)
 Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth changed callback. More...
 
static void GAS_ril_stop_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer)
 Tell solver stop notifying ATS about changes for this peers. More...
 
void * libgnunet_plugin_ats_ril_init (void *cls)
 Entry point for the plugin. More...
 
void * libgnunet_plugin_ats_ril_done (void *cls)
 Exit point for the plugin. More...
 

Detailed Description

ATS reinforcement learning solver.

Author
Fabian Oehlmann
Matthias Wachs

Definition in file plugin_ats_ril.c.

Macro Definition Documentation

◆ LOG

#define LOG (   kind,
  ... 
)    GNUNET_log_from(kind, "ats-ril", __VA_ARGS__)

◆ RIL_MIN_BW

#define RIL_MIN_BW   (5 * ntohl(GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT.value__))

◆ RIL_MAX_BW

#define RIL_MAX_BW   GNUNET_ATS_MaxBandwidth

Definition at line 38 of file plugin_ats_ril.c.

Referenced by agent_action_is_possible().

◆ RIL_ACTION_INVALID

#define RIL_ACTION_INVALID   -1

◆ RIL_INTERVAL_EXPONENT

#define RIL_INTERVAL_EXPONENT   10

Definition at line 41 of file plugin_ats_ril.c.

Referenced by ril_step_schedule_next().

◆ RIL_UTILITY_DELAY_MAX

#define RIL_UTILITY_DELAY_MAX   1000

Definition at line 42 of file plugin_ats_ril.c.

Referenced by agent_get_utility().

◆ RIL_DEFAULT_STEP_TIME_MIN

#define RIL_DEFAULT_STEP_TIME_MIN   GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 200)

Definition at line 44 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_STEP_TIME_MAX

#define RIL_DEFAULT_STEP_TIME_MAX   GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 2000)

Definition at line 45 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_ALGORITHM

#define RIL_DEFAULT_ALGORITHM   RIL_ALGO_SARSA

Definition at line 46 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_SELECT

#define RIL_DEFAULT_SELECT   RIL_SELECT_SOFTMAX

Definition at line 47 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_WELFARE

#define RIL_DEFAULT_WELFARE   RIL_WELFARE_NASH

Definition at line 48 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_DISCOUNT_BETA

#define RIL_DEFAULT_DISCOUNT_BETA   0.6

Definition at line 49 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_DISCOUNT_GAMMA

#define RIL_DEFAULT_DISCOUNT_GAMMA   0.5

Definition at line 50 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_GRADIENT_STEP_SIZE

#define RIL_DEFAULT_GRADIENT_STEP_SIZE   0.01

Definition at line 51 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TRACE_DECAY

#define RIL_DEFAULT_TRACE_DECAY   0.5

Definition at line 52 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_EXPLORE_RATIO

#define RIL_DEFAULT_EXPLORE_RATIO   1

Definition at line 53 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_EXPLORE_DECAY

#define RIL_DEFAULT_EXPLORE_DECAY   0.95

Definition at line 54 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_RBF_DIVISOR

#define RIL_DEFAULT_RBF_DIVISOR   50

Definition at line 55 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TEMPERATURE

#define RIL_DEFAULT_TEMPERATURE   0.1

Definition at line 56 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TEMPERATURE_DECAY

#define RIL_DEFAULT_TEMPERATURE_DECAY   1

Definition at line 57 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_INC_DEC_STEP_SIZE

#define RIL_INC_DEC_STEP_SIZE   1

Definition at line 59 of file plugin_ats_ril.c.

Referenced by envi_action_bw_dec(), and envi_action_bw_inc().

◆ RIL_NOP_DECAY

#define RIL_NOP_DECAY   0.5

Definition at line 60 of file plugin_ats_ril.c.

Referenced by envi_get_reward().

Enumeration Type Documentation

◆ RIL_Action_Type

ATS reinforcement learning solver.

General description The actions, how an agent can manipulate the current assignment. I.e. how the bandwidth can be changed for the currently chosen address. Not depicted in the enum are the actions of switching to a particular address. The action of switching to address with index i is depicted by the number (RIL_ACTION_TYPE_NUM + i).

Enumerator
RIL_ACTION_NOTHING 
RIL_ACTION_BW_IN_DBL 
RIL_ACTION_BW_IN_HLV 
RIL_ACTION_BW_IN_INC 
RIL_ACTION_BW_IN_DEC 
RIL_ACTION_BW_OUT_DBL 
RIL_ACTION_BW_OUT_HLV 
RIL_ACTION_BW_OUT_INC 
RIL_ACTION_BW_OUT_DEC 
RIL_ACTION_TYPE_NUM 

Definition at line 74 of file plugin_ats_ril.c.

◆ RIL_Algorithm

Enumerator
RIL_ALGO_SARSA 
RIL_ALGO_Q 

Definition at line 87 of file plugin_ats_ril.c.

87  {
88  RIL_ALGO_SARSA = 0,
89  RIL_ALGO_Q = 1
90 };

◆ RIL_Select

enum RIL_Select
Enumerator
RIL_SELECT_SOFTMAX 
RIL_SELECT_EGREEDY 

Definition at line 92 of file plugin_ats_ril.c.

◆ RIL_Welfare

Enumerator
RIL_WELFARE_NASH 
RIL_WELFARE_EGALITARIAN 

Definition at line 97 of file plugin_ats_ril.c.

◆ RIL_E_Modification

Enumerator
RIL_E_DECAY 
RIL_E_ZERO 
RIL_E_ACCUMULATE 
RIL_E_REPLACE 

Definition at line 102 of file plugin_ats_ril.c.

Function Documentation

◆ agent_q()

static double agent_q ( struct RIL_Peer_Agent agent,
const double *  state,
int  action 
)
static

Estimate the current action-value for state s and action a.

Parameters
agentagent performing the estimation
states
actiona
Returns
estimation value

Definition at line 469 of file plugin_ats_ril.c.

References RIL_Peer_Agent::m, result, and RIL_Peer_Agent::W.

Referenced by agent_get_action_max(), agent_select_softmax(), and agent_update().

472 {
473  unsigned int i;
474  double result = 0.0;
475 
476  for (i = 0; i < agent->m; i++)
477  result += state[i] * agent->W[action][i];
478 
479  /* prevent crashes if learning diverges */
480  if (isnan(result))
481  return isnan(result) * UINT32_MAX;
482  if (isinf(result))
483  return isinf(result) * UINT32_MAX;
484  return result;
485 }
unsigned int m
Number of rows of W / Number of state-vector features.
enum State state
current state of profiling
static int result
Global testing status.
double ** W
Experience matrix W.
Here is the caller graph for this function:

◆ agent_address_get_index()

static int agent_address_get_index ( struct RIL_Peer_Agent agent,
struct ATS_Address address 
)
static

Get the index of the address in the agent's list.

Parameters
agentagent handle
addressaddress handle
Returns
the index, starting with zero

Definition at line 496 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, and RIL_Address_Wrapped::next.

Referenced by agent_action_is_possible(), envi_do_action(), envi_get_state(), and GAS_ril_address_delete().

497 {
498  int i;
499  struct RIL_Address_Wrapped *cur;
500 
501  i = -1;
502  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
503  {
504  i++;
505  if (cur->address_naked == address)
506  return i;
507  }
508  return i;
509 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
struct ATS_Address * address_naked
The address.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the caller graph for this function:

◆ agent_address_get_wrapped()

static struct RIL_Address_Wrapped* agent_address_get_wrapped ( struct RIL_Peer_Agent agent,
struct ATS_Address address 
)
static

Gets the wrapped address from the agent's list.

Parameters
agentagent handle
addressaddress handle
Returns
wrapped address

Definition at line 520 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, and RIL_Address_Wrapped::next.

Referenced by GAS_ril_address_delete().

521 {
522  struct RIL_Address_Wrapped *cur;
523 
524  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
525  if (cur->address_naked == address)
526  return cur;
527  return NULL;
528 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
struct ATS_Address * address_naked
The address.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the caller graph for this function:

◆ agent_action_is_possible()

static int agent_action_is_possible ( struct RIL_Peer_Agent agent,
int  action 
)
static

Definition at line 532 of file plugin_ats_ril.c.

References ATS_Address::active, RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_assert, GNUNET_NO, GNUNET_YES, RIL_ACTION_BW_IN_DBL, RIL_ACTION_BW_IN_DEC, RIL_ACTION_BW_IN_HLV, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_DBL, RIL_ACTION_BW_OUT_DEC, RIL_ACTION_BW_OUT_HLV, RIL_ACTION_BW_OUT_INC, RIL_ACTION_NOTHING, RIL_ACTION_TYPE_NUM, and RIL_MAX_BW.

Referenced by agent_get_action_max(), agent_get_action_random(), and agent_select_softmax().

533 {
534  int address_index;
535 
536  switch (action)
537  {
538  case RIL_ACTION_NOTHING:
539  return GNUNET_YES;
540  break;
541 
544  if (agent->bw_in >= RIL_MAX_BW)
545  return GNUNET_NO;
546  else
547  return GNUNET_YES;
548  break;
549 
552  if (agent->bw_in <= 0)
553  return GNUNET_NO;
554  else
555  return GNUNET_YES;
556  break;
557 
560  if (agent->bw_out >= RIL_MAX_BW)
561  return GNUNET_NO;
562  else
563  return GNUNET_YES;
564  break;
565 
568  if (agent->bw_out <= 0)
569  return GNUNET_NO;
570  else
571  return GNUNET_YES;
572  break;
573 
574  default:
575  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) //switch address action
576  {
577  address_index = action - RIL_ACTION_TYPE_NUM;
578 
579  GNUNET_assert(address_index >= 0);
581  address_index <= agent_address_get_index(agent, agent->addresses_tail->address_naked));
582 
583  if ((agent_address_get_index(agent, agent->address_inuse) == address_index) ||
584  agent->address_inuse->active)
585  return GNUNET_NO;
586  else
587  return GNUNET_YES;
588  break;
589  }
590  // error - action does not exist
592  }
593 }
#define RIL_MAX_BW
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int active
Is this the active address for this peer?
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
struct ATS_Address * address_naked
The address.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_action_max()

static int agent_get_action_max ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Gets the action, with the maximal estimated Q-value (i.e.

the one currently estimated to bring the most reward in the future)

Parameters
agentagent performing the calculation
statethe state from which to take the action
Returns
the action promising most future reward

Definition at line 605 of file plugin_ats_ril.c.

References agent_action_is_possible(), agent_q(), GNUNET_assert, RIL_Peer_Agent::n, and RIL_ACTION_INVALID.

Referenced by agent_select_egreedy(), agent_select_softmax(), and agent_step().

606 {
607  int i;
608  int max_i = RIL_ACTION_INVALID;
609  double cur_q;
610  double max_q = -DBL_MAX;
611 
612  for (i = 0; i < agent->n; i++)
613  {
614  if (agent_action_is_possible(agent, i))
615  {
616  cur_q = agent_q(agent, state, i);
617  if (cur_q > max_q)
618  {
619  max_q = cur_q;
620  max_i = i;
621  }
622  }
623  }
624 
626 
627  return max_i;
628 }
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
enum State state
current state of profiling
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_action_random()

static int agent_get_action_random ( struct RIL_Peer_Agent agent)
static

Chooses a random action from the set of possible ones.

Parameters
agentthe agent performing the action
Returns
the action index

Definition at line 637 of file plugin_ats_ril.c.

References agent_action_is_possible(), GNUNET_assert, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::n, and RIL_ACTION_INVALID.

Referenced by agent_select_egreedy().

638 {
639  int i;
640  int is_possible[agent->n];
641  int sum = 0;
642  int r;
643 
644  for (i = 0; i < agent->n; i++)
645  {
646  if (agent_action_is_possible(agent, i))
647  {
648  is_possible[i] = GNUNET_YES;
649  sum++;
650  }
651  else
652  {
653  is_possible[i] = GNUNET_NO;
654  }
655  }
656 
658 
659  sum = -1;
660  for (i = 0; i < agent->n; i++)
661  {
662  if (is_possible[i])
663  {
664  sum++;
665  if (sum == r)
666  return i;
667  }
668  }
669 
671  return RIL_ACTION_INVALID;
672 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_YES
Definition: gnunet_common.h:77
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_update()

static void agent_update ( struct RIL_Peer_Agent agent,
double  reward,
double *  s_next,
int  a_prime 
)
static

Updates the weights (i.e.

coefficients) of the weight vector in matrix W for action a

Parameters
agentthe agent performing the update
rewardthe reward received for the last action
s_nextthe new state, the last step got the agent into
a_primethe new

Definition at line 684 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, agent_q(), RIL_Learning_Parameters::alpha, delta, RIL_Peer_Agent::E, RIL_Peer_Agent::envi, GAS_RIL_Handle::global_discount_integrated, GAS_RIL_Handle::global_discount_variable, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, RIL_Peer_Agent::s_old, and RIL_Peer_Agent::W.

Referenced by agent_step().

685 {
686  int i;
687  int k;
688  double delta;
689  double **theta = agent->W;
690 
691  delta = agent->envi->global_discount_integrated * reward; //reward
692  delta += agent->envi->global_discount_variable * agent_q(agent, s_next, a_prime); //discounted future value
693  delta -= agent_q(agent, agent->s_old, agent->a_old); //one step
694 
695 // LOG(GNUNET_ERROR_TYPE_INFO, "update() Step# %llu Q(s,a): %f a: %f r: %f y: %f Q(s+1,a+1) = %f delta: %f\n",
696 // agent->step_count,
697 // agent_q (agent, agent->s_old, agent->a_old),
698 // agent->envi->parameters.alpha,
699 // reward,
700 // agent->envi->global_discount_variable,
701 // agent_q (agent, s_next, a_prime),
702 // delta);
703 
704  for (k = 0; k < agent->n; k++)
705  {
706  for (i = 0; i < agent->m; i++)
707  {
708  // LOG(GNUNET_ERROR_TYPE_INFO, "alpha = %f delta = %f e[%d] = %f\n",
709  // agent->envi->parameters.alpha,
710  // delta,
711  // i,
712  // agent->e[i]);
713  theta[k][i] += agent->envi->parameters.alpha * delta * agent->E[k][i];
714  }
715  }
716 }
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
unsigned int m
Number of rows of W / Number of state-vector features.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
int a_old
Last chosen action.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
double ** E
Eligibility traces.
double global_discount_variable
Variable discount factor, dependent on time between steps.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
struct GAS_RIL_Handle * envi
Environment handle.
double alpha
Gradient-descent step-size.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_modify_eligibility()

static void agent_modify_eligibility ( struct RIL_Peer_Agent agent,
enum RIL_E_Modification  mod,
double *  feature,
int  action 
)
static

Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each component as in accumulating eligibility traces RIL_E_REPLACE - resets each component to feature as in replacing traces RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing.

Parameters
agentthe agent handle
modthe kind of modification
featurethe feature vector
actionthe action to take

Definition at line 732 of file plugin_ats_ril.c.

References RIL_Peer_Agent::E, RIL_Peer_Agent::envi, GAS_RIL_Handle::global_discount_variable, RIL_Learning_Parameters::lambda, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, RIL_E_ACCUMULATE, RIL_E_DECAY, RIL_E_REPLACE, and RIL_E_ZERO.

Referenced by agent_step().

736 {
737  int i;
738  int k;
739 
740  for (i = 0; i < agent->m; i++)
741  {
742  switch (mod)
743  {
744  case RIL_E_ACCUMULATE:
745  agent->E[action][i] += feature[i];
746  break;
747 
748  case RIL_E_REPLACE:
749  agent->E[action][i] = agent->E[action][i] > feature[i] ? agent->E[action][i] : feature[i];
750  break;
751 
752  case RIL_E_DECAY:
753  for (k = 0; k < agent->n; k++)
754  {
755  agent->E[k][i] *= agent->envi->global_discount_variable * agent->envi->parameters.lambda;
756  }
757  break;
758 
759  case RIL_E_ZERO:
760  for (k = 0; k < agent->n; k++)
761  {
762  agent->E[k][i] = 0;
763  }
764  break;
765  }
766  }
767 }
double lambda
Trace-decay factor for eligibility traces.
unsigned int m
Number of rows of W / Number of state-vector features.
struct RIL_Learning_Parameters parameters
Learning parameters.
double ** E
Eligibility traces.
double global_discount_variable
Variable discount factor, dependent on time between steps.
unsigned int n
Number of columns of W / Number of actions.
struct GAS_RIL_Handle * envi
Environment handle.
Here is the caller graph for this function:

◆ ril_inform()

static void ril_inform ( struct GAS_RIL_Handle solver,
enum GAS_Solver_Operation  op,
enum GAS_Solver_Status  stat 
)
static

Informs the environment about the status of the solver.

Parameters
solver
op
stat

Definition at line 777 of file plugin_ats_ril.c.

References GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, GAS_INFO_NONE, and GNUNET_ATS_PluginEnvironment::info_cb.

Referenced by ril_step().

780 {
781  solver->env->info_cb(solver->env->cls,
782  op,
783  stat,
784  GAS_INFO_NONE);
785 }
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
No more specific information.
GAS_solver_information_callback info_cb
Callback for solver to call with status information, can be NULL.
void * cls
Closure to pass to all callbacks in this struct.
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:139
Here is the caller graph for this function:

◆ ril_get_max_bw()

static unsigned long long ril_get_max_bw ( struct RIL_Scope net)
static

Calculates the maximum bandwidth an agent can assign in a network scope.

Parameters
net

Definition at line 793 of file plugin_ats_ril.c.

References RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, GNUNET_ATS_MaxBandwidth, GNUNET_MAX, and GNUNET_MIN.

Referenced by envi_action_bw_double(), envi_action_bw_inc(), and envi_get_state().

794 {
796 }
#define GNUNET_MAX(a, b)
Definition: gnunet_common.h:82
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
#define GNUNET_ATS_MaxBandwidth
Maximum bandwidth assigned to a network : 4095 MB/s.
uint32_t bw_out_available
Total available outbound bandwidth.
uint32_t bw_in_available
Total available inbound bandwidth.
Here is the caller graph for this function:

◆ envi_set_active_suggestion()

static void envi_set_active_suggestion ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
struct ATS_Address new_address,
unsigned long long  new_bw_in,
unsigned long long  new_bw_out,
int  silent 
)
static

Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify ATS of its new decision.

Parameters
solversolver handle
agentagent handle
new_addressthe address which is to be used
new_bw_inthe new amount of inbound bandwidth set for this address
new_bw_outthe new amount of outbound bandwidth set for this address
silentdisables invocation of the bw_changed callback, if GNUNET_YES

Definition at line 810 of file plugin_ats_ril.c.

References ATS_Address::active, RIL_Peer_Agent::address_inuse, ATS_Address::assigned_bw_in, ATS_Address::assigned_bw_out, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::is_active, LOG, notify, RIL_Peer_Agent::peer, RIL_Peer_Agent::suggestion_address, and RIL_Peer_Agent::suggestion_issue.

Referenced by envi_action_address_switch(), envi_action_bw_dec(), envi_action_bw_double(), envi_action_bw_halven(), envi_action_bw_inc(), GAS_ril_address_delete(), GAS_ril_get_preferred_address(), GAS_ril_stop_get_preferred_address(), and ril_try_unblock_agent().

816 {
817  int notify = GNUNET_NO;
818 
820  " set_active_suggestion() for peer '%s'\n",
821  GNUNET_i2s(&agent->peer));
822 
823  //address change
824  if (agent->address_inuse != new_address)
825  {
826  if (NULL != agent->address_inuse)
827  {
828  agent->address_inuse->active = GNUNET_NO;
829  agent->address_inuse->assigned_bw_in = 0;
830  agent->address_inuse->assigned_bw_out = 0;
831  }
832  if (NULL != new_address)
833  {
834  LOG(GNUNET_ERROR_TYPE_DEBUG, " set address active: %s\n", agent->is_active ? "yes" : "no");
835  new_address->active = agent->is_active;
836  new_address->assigned_bw_in = agent->bw_in;
837  new_address->assigned_bw_out = agent->bw_out;
838  }
839  notify |= GNUNET_YES;
840  }
841 
842  if (new_address)
843  {
844  //activity change
845  if (new_address->active != agent->is_active)
846  {
847  new_address->active = agent->is_active;
848  notify |= GNUNET_YES;
849  }
850 
851  //bw change
852  if (agent->bw_in != new_bw_in)
853  {
854  agent->bw_in = new_bw_in;
855  new_address->assigned_bw_in = new_bw_in;
856  notify |= GNUNET_YES;
857  }
858  if (agent->bw_out != new_bw_out)
859  {
860  agent->bw_out = new_bw_out;
861  new_address->assigned_bw_out = new_bw_out;
862  notify |= GNUNET_YES;
863  }
864  }
865 
866  if (notify && agent->is_active && (GNUNET_NO == silent))
867  {
868  if (new_address)
869  {
870  LOG(GNUNET_ERROR_TYPE_DEBUG, " envi_set_active_suggestion() notify\n");
871  agent->suggestion_issue = GNUNET_YES;
872  agent->suggestion_address = new_address;
873  }
874  else if (agent->address_inuse)
875  {
876  /* disconnect case, no new address */
879  agent->bw_in = 0;
880  agent->bw_out = 0;
881 
882  agent->suggestion_issue = GNUNET_YES;
883  agent->suggestion_address = agent->address_inuse;
884  }
885  }
886  agent->address_inuse = new_address;
887 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
int active
Is this the active address for this peer?
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
int suggestion_issue
Flag whether a suggestion has to be issued.
uint32_t assigned_bw_in
Inbound bandwidth assigned by solver.
struct GNUNET_PeerIdentity peer
Peer ID.
static int silent
–silent option
Definition: gnunet-qr.c:49
static struct GNUNET_PEERINFO_NotifyContext * notify
Handle to the peerinfo notify service (NULL until we&#39;ve connected to it).
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
uint32_t assigned_bw_out
Outbound bandwidth assigned by solver.
#define GNUNET_YES
Definition: gnunet_common.h:77
#define LOG(kind,...)
struct ATS_Address * suggestion_address
The address which has to be issued.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_get_state()

static double* envi_get_state ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Allocates a state vector and fills it with the features present.

Parameters
solverthe solver handle
agentthe agent handle
Returns
pointer to the state vector

Definition at line 897 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, agent_address_get_index(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, removetrailingwhitespace::f, GNUNET_malloc, m, RIL_Peer_Agent::m, GAS_RIL_Handle::parameters, RIL_Learning_Parameters::rbf_divisor, ril_get_max_bw(), ATS_Address::solver_information, and state.

Referenced by agent_step().

898 {
899  double *state;
900  double y[2];
901  double x[2];
902  double d[2];
903  double sigma;
904  double f;
905  int m;
906  int i;
907  int k;
908  unsigned long long max_bw;
909 
910  state = GNUNET_malloc(sizeof(double) * agent->m);
911 
912  max_bw = ril_get_max_bw((struct RIL_Scope *)agent->address_inuse->solver_information);
913 
914  y[0] = (double)agent->bw_out;
915  y[1] = (double)agent->bw_in;
916 
917  m = agent_address_get_index(agent, agent->address_inuse) * (solver->parameters.rbf_divisor + 1) * (solver->parameters.rbf_divisor + 1);
918  for (i = 0; i <= solver->parameters.rbf_divisor; i++)
919  {
920  for (k = 0; k <= solver->parameters.rbf_divisor; k++)
921  {
922  x[0] = (double)i * (double)max_bw / (double)solver->parameters.rbf_divisor;
923  x[1] = (double)k * (double)max_bw / (double)solver->parameters.rbf_divisor;
924  d[0] = x[0] - y[0];
925  d[1] = x[1] - y[1];
926  sigma = (((double)max_bw / ((double)solver->parameters.rbf_divisor + 1)) * 0.5);
927  f = exp(-((d[0] * d[0] + d[1] * d[1]) / (2 * sigma * sigma)));
928  state[m++] = f;
929  }
930  }
931 
932  return state;
933 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
unsigned int m
Number of rows of W / Number of state-vector features.
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:99
enum State state
current state of profiling
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
struct RIL_Learning_Parameters parameters
Learning parameters.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
unsigned long long rbf_divisor
State space divisor.
#define GNUNET_malloc(size)
Wrapper around malloc.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_utility()

static double agent_get_utility ( struct RIL_Peer_Agent agent)
static

Returns the utility value of the connection an agent manages.

Parameters
agentthe agent in question
Returns
the utility value

Definition at line 943 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, RIL_Peer_Agent::envi, GNUNET_ATS_PluginEnvironment::get_preferences, GNUNET_ATS_PREFERENCE_BANDWIDTH, GNUNET_ATS_PREFERENCE_LATENCY, GAS_NormalizationInfo::norm, ATS_Address::norm_delay, RIL_Peer_Agent::peer, RIL_MIN_BW, and RIL_UTILITY_DELAY_MAX.

Referenced by envi_get_reward(), and ril_network_get_social_welfare().

944 {
945  const double *preferences;
946  double delay_atsi;
947  double delay_norm;
948  double pref_match;
949 
950  preferences = agent->envi->env->get_preferences(agent->envi->env->cls,
951  &agent->peer);
952 
953  delay_atsi = agent->address_inuse->norm_delay.norm;
954  delay_norm = RIL_UTILITY_DELAY_MAX * exp(-delay_atsi * 0.00001);
955 
956  pref_match = preferences[GNUNET_ATS_PREFERENCE_LATENCY] * delay_norm;
957  pref_match += preferences[GNUNET_ATS_PREFERENCE_BANDWIDTH] *
958  sqrt((double)(agent->bw_in / RIL_MIN_BW) * (double)(agent->bw_out / RIL_MIN_BW));
959  return pref_match;
960 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
#define RIL_UTILITY_DELAY_MAX
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
struct GNUNET_PeerIdentity peer
Peer ID.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Change the peer&#39;s latency value to the given amount.
Change the peer&#39;s bandwidth value (value per byte of bandwidth in the goal function) to the given amo...
struct GAS_NormalizationInfo norm_delay
Normalized delay information for this address.
void * cls
Closure to pass to all callbacks in this struct.
double norm
Normalized values from queue to a range of values [1.0...2.0].
struct GAS_RIL_Handle * envi
Environment handle.
#define RIL_MIN_BW
Here is the caller graph for this function:

◆ ril_network_get_social_welfare()

static double ril_network_get_social_welfare ( struct GAS_RIL_Handle solver,
struct RIL_Scope scope 
)
static

Calculates the social welfare within a network scope according to what social welfare measure is set in the configuration.

Parameters
solverthe solver handle
scopethe network scope in question
Returns
the social welfare value

Definition at line 971 of file plugin_ats_ril.c.

References RIL_Scope::active_agent_count, RIL_Peer_Agent::address_inuse, agent_get_utility(), GAS_RIL_Handle::agents_head, GNUNET_assert, GNUNET_MIN, GNUNET_NO, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, GAS_RIL_Handle::parameters, result, RIL_WELFARE_EGALITARIAN, RIL_WELFARE_NASH, RIL_Learning_Parameters::social_welfare, and ATS_Address::solver_information.

Referenced by ril_networks_update_state().

972 {
973  struct RIL_Peer_Agent *cur;
974  double result;
975 
976  switch (solver->parameters.social_welfare)
977  {
979  result = DBL_MAX;
980  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
981  {
982  if (cur->is_active && cur->address_inuse && (cur->address_inuse->solver_information == scope))
983  {
984  result = GNUNET_MIN(result, agent_get_utility(cur));
985  }
986  }
987  return result;
988 
989  case RIL_WELFARE_NASH:
990  result = 0;
991  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
992  {
993  if (cur->is_active && cur->address_inuse && (cur->address_inuse->solver_information == scope))
994  {
995  result *= pow(agent_get_utility(cur), 1.0 / (double)scope->active_agent_count);
996  }
997  }
998  return result;
999  }
1001  return 1;
1002 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
static int result
Global testing status.
struct RIL_Learning_Parameters parameters
Learning parameters.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
unsigned int active_agent_count
Number of active agents in scope.
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_get_penalty()

static double envi_get_penalty ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Definition at line 1005 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_INC, RIL_MIN_BW, and ATS_Address::solver_information.

Referenced by envi_get_reward().

1006 {
1007  struct RIL_Scope *net;
1008  unsigned long long over_max;
1009  unsigned long long over_in = 0;
1010  unsigned long long over_out = 0;
1011 
1012  net = agent->address_inuse->solver_information;
1013 
1014  if (net->bw_in_utilized > net->bw_in_available)
1015  {
1016  over_in = net->bw_in_utilized - net->bw_in_available;
1017  if (RIL_ACTION_BW_IN_INC == agent->a_old)
1018  {
1019  /* increase quadratically */
1020  over_in *= over_in;
1021  }
1022  }
1023  if (net->bw_out_utilized > net->bw_out_available)
1024  {
1025  over_out = net->bw_out_utilized - net->bw_out_available;
1026  if (RIL_ACTION_BW_OUT_INC == agent->a_old)
1027  {
1028  /* increase quadratically */
1029  over_out *= over_out;
1030  }
1031  }
1032  over_max = (over_in + over_out) / (RIL_MIN_BW * RIL_MIN_BW);
1033 
1034  return -1.0 * (double)over_max;
1035 }
int a_old
Last chosen action.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the caller graph for this function:

◆ envi_get_reward()

static double envi_get_reward ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Gets the reward for the last performed step, which is calculated in equal parts from the local (the peer specific) and the global (for all peers identical) reward.

Parameters
solverthe solver handle
agentthe agent handle
Returns
the reward

Definition at line 1047 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, agent_get_utility(), delta, envi_get_penalty(), RIL_Peer_Agent::nop_bonus, RIL_Peer_Agent::objective_old, RIL_ACTION_NOTHING, RIL_NOP_DECAY, RIL_Scope::social_welfare, and ATS_Address::solver_information.

Referenced by agent_step().

1048 {
1049  struct RIL_Scope *net;
1050  double objective;
1051  double delta;
1052  double steady;
1053  double penalty;
1054  double reward;
1055 
1056  net = agent->address_inuse->solver_information;
1057 
1058  penalty = envi_get_penalty(solver, agent);
1059  objective = (agent_get_utility(agent) + net->social_welfare) / 2;
1060  delta = objective - agent->objective_old;
1061  agent->objective_old = objective;
1062 
1063  if (delta != 0 && penalty == 0)
1064  {
1065  agent->nop_bonus = delta * RIL_NOP_DECAY;
1066  }
1067  else
1068  {
1069  agent->nop_bonus *= RIL_NOP_DECAY;
1070  }
1071 
1072  steady = (RIL_ACTION_NOTHING == agent->a_old) ? agent->nop_bonus : 0;
1073 
1074  reward = delta + steady;
1075  return reward + penalty;
1076 }
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
int a_old
Last chosen action.
double social_welfare
The social welfare achieved in the scope.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
#define RIL_NOP_DECAY
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
static double envi_get_penalty(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_double()

static void envi_action_bw_double ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Doubles the bandwidth for the active address.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise the outbound bandwidth

Definition at line 1086 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, ril_get_max_bw(), and ATS_Address::solver_information.

Referenced by envi_do_action().

1089 {
1090  unsigned long long new_bw;
1091  unsigned long long max_bw;
1092 
1093  max_bw = ril_get_max_bw((struct RIL_Scope *)agent->address_inuse->solver_information);
1094 
1095  if (direction_in)
1096  {
1097  new_bw = agent->bw_in * 2;
1098  if (new_bw < agent->bw_in || new_bw > max_bw)
1099  new_bw = max_bw;
1100  envi_set_active_suggestion(solver, agent, agent->address_inuse, new_bw,
1101  agent->bw_out, GNUNET_NO);
1102  }
1103  else
1104  {
1105  new_bw = agent->bw_out * 2;
1106  if (new_bw < agent->bw_out || new_bw > max_bw)
1107  new_bw = max_bw;
1108  envi_set_active_suggestion(solver, agent, agent->address_inuse, agent->bw_in,
1109  new_bw, GNUNET_NO);
1110  }
1111 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_halven()

static void envi_action_bw_halven ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Cuts the bandwidth for the active address in half.

The least amount of bandwidth suggested, is the minimum bandwidth for a peer, in order to not invoke a disconnect.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1123 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), and GNUNET_NO.

Referenced by envi_do_action().

1126 {
1127  unsigned long long new_bw;
1128 
1129  if (direction_in)
1130  {
1131  new_bw = agent->bw_in / 2;
1132  if (new_bw <= 0 || new_bw > agent->bw_in)
1133  new_bw = 0;
1134  envi_set_active_suggestion(solver, agent, agent->address_inuse, new_bw, agent->bw_out,
1135  GNUNET_NO);
1136  }
1137  else
1138  {
1139  new_bw = agent->bw_out / 2;
1140  if (new_bw <= 0 || new_bw > agent->bw_out)
1141  new_bw = 0;
1142  envi_set_active_suggestion(solver, agent, agent->address_inuse, agent->bw_in, new_bw,
1143  GNUNET_NO);
1144  }
1145 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_inc()

static void envi_action_bw_inc ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Increases the bandwidth by 5 times the minimum bandwidth for the active address.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1156 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, ril_get_max_bw(), RIL_INC_DEC_STEP_SIZE, RIL_MIN_BW, and ATS_Address::solver_information.

Referenced by envi_do_action().

1157 {
1158  unsigned long long new_bw;
1159  unsigned long long max_bw;
1160 
1161  max_bw = ril_get_max_bw((struct RIL_Scope *)agent->address_inuse->solver_information);
1162 
1163  if (direction_in)
1164  {
1165  new_bw = agent->bw_in + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1166  if (new_bw < agent->bw_in || new_bw > max_bw)
1167  new_bw = max_bw;
1168  envi_set_active_suggestion(solver, agent, agent->address_inuse, new_bw,
1169  agent->bw_out, GNUNET_NO);
1170  }
1171  else
1172  {
1173  new_bw = agent->bw_out + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1174  if (new_bw < agent->bw_out || new_bw > max_bw)
1175  new_bw = max_bw;
1176  envi_set_active_suggestion(solver, agent, agent->address_inuse, agent->bw_in,
1177  new_bw, GNUNET_NO);
1178  }
1179 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define RIL_INC_DEC_STEP_SIZE
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_dec()

static void envi_action_bw_dec ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Decreases the bandwidth by 5 times the minimum bandwidth for the active address.

The least amount of bandwidth suggested, is the minimum bandwidth for a peer, in order to not invoke a disconnect.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1191 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, RIL_INC_DEC_STEP_SIZE, and RIL_MIN_BW.

Referenced by envi_do_action().

1192 {
1193  unsigned long long new_bw;
1194 
1195  if (direction_in)
1196  {
1197  new_bw = agent->bw_in - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1198  if (new_bw <= 0 || new_bw > agent->bw_in)
1199  new_bw = 0;
1200  envi_set_active_suggestion(solver, agent, agent->address_inuse, new_bw, agent->bw_out,
1201  GNUNET_NO);
1202  }
1203  else
1204  {
1205  new_bw = agent->bw_out - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1206  if (new_bw <= 0 || new_bw > agent->bw_out)
1207  new_bw = 0;
1208  envi_set_active_suggestion(solver, agent, agent->address_inuse, agent->bw_in, new_bw,
1209  GNUNET_NO);
1210  }
1211 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define RIL_INC_DEC_STEP_SIZE
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_address_switch()

static void envi_action_address_switch ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
unsigned int  address_index 
)
static

Switches to the address given by its index.

Parameters
solversolver handle
agentagent handle
address_indexindex of the address as it is saved in the agent's list, starting with zero

Definition at line 1221 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_assert, GNUNET_NO, and RIL_Address_Wrapped::next.

Referenced by envi_do_action().

1224 {
1225  struct RIL_Address_Wrapped *cur;
1226  int i = 0;
1227 
1228  //cur = agent_address_get_wrapped(agent, agent->address_inuse);
1229 
1230  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
1231  {
1232  if (i == address_index)
1233  {
1234  envi_set_active_suggestion(solver, agent, cur->address_naked, agent->bw_in, agent->bw_out,
1235  GNUNET_NO);
1236  return;
1237  }
1238 
1239  i++;
1240  }
1241 
1242  //no address with address_index exists, in this case this action should not be callable
1244 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_do_action()

static void envi_do_action ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  action 
)
static

Puts the action into effect by calling the according function.

Parameters
solverthe solver handle
agentthe action handle
actionthe action to perform by the solver

Definition at line 1254 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), envi_action_address_switch(), envi_action_bw_dec(), envi_action_bw_double(), envi_action_bw_halven(), envi_action_bw_inc(), GNUNET_assert, GNUNET_NO, GNUNET_YES, RIL_ACTION_BW_IN_DBL, RIL_ACTION_BW_IN_DEC, RIL_ACTION_BW_IN_HLV, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_DBL, RIL_ACTION_BW_OUT_DEC, RIL_ACTION_BW_OUT_HLV, RIL_ACTION_BW_OUT_INC, RIL_ACTION_NOTHING, and RIL_ACTION_TYPE_NUM.

Referenced by agent_step().

1255 {
1256  int address_index;
1257 
1258  switch (action)
1259  {
1260  case RIL_ACTION_NOTHING:
1261  break;
1262 
1263  case RIL_ACTION_BW_IN_DBL:
1264  envi_action_bw_double(solver, agent, GNUNET_YES);
1265  break;
1266 
1267  case RIL_ACTION_BW_IN_HLV:
1268  envi_action_bw_halven(solver, agent, GNUNET_YES);
1269  break;
1270 
1271  case RIL_ACTION_BW_IN_INC:
1272  envi_action_bw_inc(solver, agent, GNUNET_YES);
1273  break;
1274 
1275  case RIL_ACTION_BW_IN_DEC:
1276  envi_action_bw_dec(solver, agent, GNUNET_YES);
1277  break;
1278 
1279  case RIL_ACTION_BW_OUT_DBL:
1280  envi_action_bw_double(solver, agent, GNUNET_NO);
1281  break;
1282 
1283  case RIL_ACTION_BW_OUT_HLV:
1284  envi_action_bw_halven(solver, agent, GNUNET_NO);
1285  break;
1286 
1287  case RIL_ACTION_BW_OUT_INC:
1288  envi_action_bw_inc(solver, agent, GNUNET_NO);
1289  break;
1290 
1291  case RIL_ACTION_BW_OUT_DEC:
1292  envi_action_bw_dec(solver, agent, GNUNET_NO);
1293  break;
1294 
1295  default:
1296  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) //switch address action
1297  {
1298  address_index = action - RIL_ACTION_TYPE_NUM;
1299 
1300  GNUNET_assert(address_index >= 0);
1301  GNUNET_assert(
1302  address_index <= agent_address_get_index(agent, agent->addresses_tail->address_naked));
1303 
1304  envi_action_address_switch(solver, agent, address_index);
1305  break;
1306  }
1307  // error - action does not exist
1309  }
1310 }
static void envi_action_bw_halven(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Cuts the bandwidth for the active address in half.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void envi_action_bw_dec(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Decreases the bandwidth by 5 times the minimum bandwidth for the active address.
#define GNUNET_NO
Definition: gnunet_common.h:78
struct ATS_Address * address_naked
The address.
static void envi_action_address_switch(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
Switches to the address given by its index.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
static void envi_action_bw_double(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Doubles the bandwidth for the active address.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
static void envi_action_bw_inc(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Increases the bandwidth by 5 times the minimum bandwidth for the active address.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_egreedy()

static int agent_select_egreedy ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Selects the next action using the e-greedy strategy.

I.e. with a probability of (1-e) the action with the maximum expected return will be chosen (=> exploitation) and with probability (e) a random action will be chosen. In case the Q-learning rule is set, the function also resets the eligibility traces in the exploration case (after Watkin's Q-learning).

Parameters
agentthe agent selecting an action
statethe current state-feature vector
Returns
the action index

Definition at line 1324 of file plugin_ats_ril.c.

References agent_get_action_max(), agent_get_action_random(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_decay, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_YES, GAS_RIL_Handle::parameters, and RIL_ALGO_Q.

Referenced by agent_select_action().

1325 {
1326  int action;
1328  UINT32_MAX) / (double)UINT32_MAX;
1329 
1330  if (r < agent->envi->parameters.epsilon) //explore
1331  {
1332  action = agent_get_action_random(agent);
1333  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1334  {
1335  agent->eligibility_reset = GNUNET_YES;
1336  }
1337  agent->envi->parameters.epsilon *= agent->envi->parameters.epsilon_decay;
1338  return action;
1339  }
1340  else //exploit
1341  {
1342  action = agent_get_action_max(agent, state);
1343  return action;
1344  }
1345 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int agent_get_action_random(struct RIL_Peer_Agent *agent)
Chooses a random action from the set of possible ones.
enum State state
current state of profiling
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_decay
Decay factor of the explore ratio.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_softmax()

static int agent_select_softmax ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Selects the next action with a probability corresponding to its value.

The probability is calculated using a Boltzmann distribution with a temperature value. The higher the temperature, the more are the action selection probabilities the same. With a temperature of 0, the selection is greedy, i.e. always the action with the highest value is chosen.

Parameters
agent
state
Returns

Definition at line 1358 of file plugin_ats_ril.c.

References agent_action_is_possible(), agent_get_action_max(), agent_q(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, GNUNET_assert, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::n, p, GAS_RIL_Handle::parameters, RIL_ACTION_INVALID, RIL_ALGO_Q, RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_decay.

Referenced by agent_select_action().

1359 {
1360  int i;
1361  int a_max;
1362  double eqt[agent->n];
1363  double p[agent->n];
1364  double sum = 0;
1365  double r;
1366 
1367  a_max = agent_get_action_max(agent, state);
1368 
1369  for (i = 0; i < agent->n; i++)
1370  {
1371  if (agent_action_is_possible(agent, i))
1372  {
1373  eqt[i] = exp(agent_q(agent, state, i) / agent->envi->parameters.temperature);
1374  if (isinf(eqt[i]))
1375  eqt[i] = isinf(eqt[i]) * UINT32_MAX;
1376  sum += eqt[i];
1377  }
1378  }
1379  for (i = 0; i < agent->n; i++)
1380  {
1381  if (agent_action_is_possible(agent, i))
1382  {
1383  p[i] = eqt[i] / sum;
1384  }
1385  else
1386  {
1387  p[i] = 0;
1388  }
1389  }
1391  UINT32_MAX) / (double)UINT32_MAX;
1392  sum = 0;
1393  for (i = 0; i < agent->n; i++)
1394  {
1395  if (sum + p[i] > r)
1396  {
1397  if (i != a_max)
1398  {
1399  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1400  agent->eligibility_reset = GNUNET_YES;
1402  }
1403  return i;
1404  }
1405  sum += p[i];
1406  }
1408  return RIL_ACTION_INVALID;
1409 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double temperature_decay
Decay factor of the temperature value.
#define GNUNET_NO
Definition: gnunet_common.h:78
double temperature
Softmax action-selection temperature.
enum State state
current state of profiling
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_YES
Definition: gnunet_common.h:77
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_action()

static int agent_select_action ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Select the next action of an agent either according to the e-greedy strategy or the softmax strategy.

Parameters
agentthe agent in question
statethe current state-feature vector
Returns
the action index

Definition at line 1420 of file plugin_ats_ril.c.

References agent_select_egreedy(), agent_select_softmax(), RIL_Peer_Agent::envi, GAS_RIL_Handle::parameters, RIL_SELECT_EGREEDY, and RIL_Learning_Parameters::select.

Referenced by agent_step().

1421 {
1422  if (agent->envi->parameters.select == RIL_SELECT_EGREEDY)
1423  {
1424  return agent_select_egreedy(agent, state);
1425  }
1426  else
1427  {
1428  return agent_select_softmax(agent, state);
1429  }
1430 }
static int agent_select_softmax(struct RIL_Peer_Agent *agent, double *state)
Selects the next action with a probability corresponding to its value.
enum State state
current state of profiling
enum RIL_Select select
Action selection strategy;.
static int agent_select_egreedy(struct RIL_Peer_Agent *agent, double *state)
Selects the next action using the e-greedy strategy.
struct RIL_Learning_Parameters parameters
Learning parameters.
struct GAS_RIL_Handle * envi
Environment handle.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_step()

static void agent_step ( struct RIL_Peer_Agent agent)
static

Performs one step of the Markov Decision Process.

Other than in the literature the step starts after having done the last action a_old. It observes the new state s_next and the reward received. Then the coefficient update is done according to the SARSA or Q-learning method. The next action is put into effect.

Parameters
agentthe agent performing the step

Definition at line 1441 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, agent_get_action_max(), agent_modify_eligibility(), agent_select_action(), agent_update(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::eligibility_reset, RIL_Learning_Parameters::eligibility_trace_mode, RIL_Peer_Agent::envi, envi_do_action(), envi_get_reward(), envi_get_state(), GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_i2s(), GNUNET_NO, LOG, GAS_RIL_Handle::parameters, RIL_Peer_Agent::peer, RIL_ACTION_INVALID, RIL_ALGO_Q, RIL_ALGO_SARSA, RIL_E_DECAY, RIL_E_ZERO, ril_step(), RIL_Peer_Agent::s_old, and RIL_Peer_Agent::step_count.

Referenced by ril_step().

1442 {
1443  int a_next = RIL_ACTION_INVALID;
1444  int a_max;
1445  double *s_next;
1446  double reward;
1447 
1448  LOG(GNUNET_ERROR_TYPE_DEBUG, " agent_step() Peer '%s', algorithm %s\n",
1449  GNUNET_i2s(&agent->peer),
1450  agent->envi->parameters.algorithm ? "Q" : "SARSA");
1451 
1452  s_next = envi_get_state(agent->envi, agent);
1453  reward = envi_get_reward(agent->envi, agent);
1454 
1455  if (agent->eligibility_reset)
1456  {
1457  agent_modify_eligibility(agent, RIL_E_ZERO, NULL, -1);
1458  agent->eligibility_reset = GNUNET_NO;
1459  }
1460  else
1461  {
1462  agent_modify_eligibility(agent, RIL_E_DECAY, NULL, -1);
1463  }
1464  if (RIL_ACTION_INVALID != agent->a_old)
1465  {
1467  }
1468 
1469  switch (agent->envi->parameters.algorithm)
1470  {
1471  case RIL_ALGO_SARSA:
1472  a_next = agent_select_action(agent, s_next);
1473  if (RIL_ACTION_INVALID != agent->a_old)
1474  {
1475  //updates weights with selected action (on-policy), if not first step
1476  agent_update(agent, reward, s_next, a_next);
1477  }
1478  break;
1479 
1480  case RIL_ALGO_Q:
1481  a_max = agent_get_action_max(agent, s_next);
1482  if (RIL_ACTION_INVALID != agent->a_old)
1483  {
1484  //updates weights with best action, disregarding actually selected action (off-policy), if not first step
1485  agent_update(agent, reward, s_next, a_max);
1486  }
1487  a_next = agent_select_action(agent, s_next);
1488  break;
1489  }
1490 
1491  GNUNET_assert(RIL_ACTION_INVALID != a_next);
1492 
1493  LOG(GNUNET_ERROR_TYPE_DEBUG, "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n",
1494  agent->step_count,
1495  reward,
1496  agent->bw_in / 1024,
1497  agent->bw_out / 1024,
1498  a_next);
1499 
1500  envi_do_action(agent->envi, agent, a_next);
1501 
1502  GNUNET_free(agent->s_old);
1503  agent->s_old = s_next;
1504  agent->a_old = a_next;
1505 
1506  agent->step_count += 1;
1507 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
int a_old
Last chosen action.
#define GNUNET_NO
Definition: gnunet_common.h:78
static double * envi_get_state(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Allocates a state vector and fills it with the features present.
struct GNUNET_PeerIdentity peer
Peer ID.
static void envi_do_action(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
Puts the action into effect by calling the according function.
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
#define RIL_ACTION_INVALID
static void agent_update(struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
Updates the weights (i.e.
static double envi_get_reward(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Gets the reward for the last performed step, which is calculated in equal parts from the local (the p...
static int agent_select_action(struct RIL_Peer_Agent *agent, double *state)
Select the next action of an agent either according to the e-greedy strategy or the softmax strategy...
double * s_old
Last perceived state feature vector.
#define LOG(kind,...)
static void agent_modify_eligibility(struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each co...
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
struct GAS_RIL_Handle * envi
Environment handle.
#define GNUNET_free(ptr)
Wrapper around free.
unsigned long long step_count
Number of performed time-steps.
enum RIL_E_Modification eligibility_trace_mode
Whether to accumulate or replace eligibility traces.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step()

static void ril_step ( struct GAS_RIL_Handle solver)
static

Prototype of the ril_step() procedure.

Triggers one step per agent.

Parameters
solverthe solver handle
solver

Definition at line 1871 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, agent_step(), GAS_RIL_Handle::agents_head, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, GAS_RIL_Handle::bulk_changes, GAS_RIL_Handle::bulk_lock, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, GAS_OP_SOLVE_START, GAS_OP_SOLVE_STOP, GAS_OP_SOLVE_UPDATE_NOTIFICATION_START, GAS_OP_SOLVE_UPDATE_NOTIFICATION_STOP, GAS_STAT_SUCCESS, GNUNET_ERROR_TYPE_DEBUG, GNUNET_NO, GNUNET_TIME_absolute_get(), GNUNET_YES, RIL_Peer_Agent::is_active, LOG, RIL_Peer_Agent::next, ril_calculate_discount(), ril_inform(), ril_networks_update_state(), ril_step_schedule_next(), ril_try_unblock_agent(), GAS_RIL_Handle::step_count, GAS_RIL_Handle::step_time_last, RIL_Peer_Agent::suggestion_address, and RIL_Peer_Agent::suggestion_issue.

Referenced by agent_step(), GAS_ril_address_add(), GAS_ril_address_change_preference(), GAS_ril_address_delete(), GAS_ril_address_property_changed(), GAS_ril_bulk_stop(), GAS_ril_stop_get_preferred_address(), and ril_step_scheduler_task().

1872 {
1873  struct RIL_Peer_Agent *cur;
1874 
1875  if (GNUNET_YES == solver->bulk_lock)
1876  {
1877  solver->bulk_changes++;
1878  return;
1879  }
1880 
1882 
1883  LOG(GNUNET_ERROR_TYPE_DEBUG, " RIL step number %d\n", solver->step_count);
1884 
1885  if (0 == solver->step_count)
1886  {
1888  }
1889 
1890  ril_calculate_discount(solver);
1891  ril_networks_update_state(solver);
1892 
1893  //trigger one step per active, unblocked agent
1894  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1895  {
1896  if (cur->is_active)
1897  {
1898  if (NULL == cur->address_inuse)
1899  {
1900  ril_try_unblock_agent(solver, cur, GNUNET_NO);
1901  }
1902  if (cur->address_inuse)
1903  {
1904  agent_step(cur);
1905  }
1906  }
1907  }
1908 
1909  ril_networks_update_state(solver);
1910 
1911  solver->step_count++;
1912  ril_step_schedule_next(solver);
1913 
1915 
1917  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1918  {
1919  if (cur->suggestion_issue)
1920  {
1921  solver->env->bandwidth_changed_cb(solver->env->cls,
1922  cur->suggestion_address);
1923  cur->suggestion_issue = GNUNET_NO;
1924  }
1925  }
1927 }
After the problem was finished, start notifications about changes to addresses.
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
int bulk_changes
Number of changes during a lock.
static void ril_inform(struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
Informs the environment about the status of the solver.
#define GNUNET_NO
Definition: gnunet_common.h:78
int suggestion_issue
Flag whether a suggestion has to be issued.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
unsigned long long step_count
Number of performed steps.
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
A solution iteration has been started.
static void ril_step_schedule_next(struct GAS_RIL_Handle *solver)
Schedules the next global step in an adaptive way.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
static void ril_calculate_discount(struct GAS_RIL_Handle *solver)
Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.
struct ATS_Address * address_inuse
Address in use.
A solution iteration has been finished.
static void agent_step(struct RIL_Peer_Agent *agent)
Performs one step of the Markov Decision Process.
static void ril_networks_update_state(struct GAS_RIL_Handle *solver)
Retrieves the state of the network scope, so that its attributes are up-to- date. ...
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:77
int bulk_lock
Lock for bulk operations.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
#define LOG(kind,...)
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
After the problem was finished, notifications about changes to addresses are done.
struct ATS_Address * suggestion_address
The address which has to be issued.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step_scheduler_task()

static void ril_step_scheduler_task ( void *  cls)
static

Task for the scheduler, which performs one step and lets the solver know that no further step is scheduled.

Parameters
clsthe solver handle

Definition at line 1525 of file plugin_ats_ril.c.

References ril_step(), and GAS_RIL_Handle::step_next_task_id.

Referenced by ril_step_schedule_next().

1526 {
1527  struct GAS_RIL_Handle *solver = cls;
1528 
1529  solver->step_next_task_id = NULL;
1530  ril_step(solver);
1531 }
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
A handle for the reinforcement learning solver.
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_get_used_resource_ratio()

static double ril_get_used_resource_ratio ( struct GAS_RIL_Handle solver)
static

Determines how much of the available bandwidth is assigned.

If more is assigned than available it returns 1. The function is used to determine the step size of the adaptive stepping.

Parameters
solverthe solver handle
Returns
the ratio

Definition at line 1542 of file plugin_ats_ril.c.

References RIL_Scope::bw_in_assigned, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, GAS_RIL_Handle::network_entries, and GAS_RIL_Handle::networks_count.

Referenced by ril_step_schedule_next().

1543 {
1544  int i;
1545  struct RIL_Scope net;
1546  unsigned long long sum_assigned = 0;
1547  unsigned long long sum_available = 0;
1548  double ratio;
1549 
1550  for (i = 0; i < solver->networks_count; i++)
1551  {
1552  net = solver->network_entries[i];
1553  if (net.bw_in_assigned > 0) //only consider scopes where an address is actually active
1554  {
1555  sum_assigned += net.bw_in_utilized;
1556  sum_assigned += net.bw_out_utilized;
1557  sum_available += net.bw_in_available;
1558  sum_available += net.bw_out_available;
1559  }
1560  }
1561  if (sum_available > 0)
1562  {
1563  ratio = ((double)sum_assigned) / ((double)sum_available);
1564  }
1565  else
1566  {
1567  ratio = 0;
1568  }
1569 
1570  return ratio > 1 ? 1 : ratio; //overutilization is possible, cap at 1
1571 }
struct RIL_Scope * network_entries
Array of networks with global assignment state.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
unsigned int networks_count
Networks count.
Here is the caller graph for this function:

◆ ril_get_network()

static struct RIL_Scope* ril_get_network ( struct GAS_RIL_Handle s,
uint32_t  type 
)
static

Lookup network struct by type.

Parameters
sthe solver handle
typethe network type
Returns
the network struct

Definition at line 1581 of file plugin_ats_ril.c.

References GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, and RIL_Scope::type.

Referenced by GAS_ril_address_add(), ril_network_is_active(), and ril_network_is_not_full().

1582 {
1583  int i;
1584 
1585  for (i = 0; i < s->networks_count; i++)
1586  {
1587  if (s->network_entries[i].type == type)
1588  {
1589  return &s->network_entries[i];
1590  }
1591  }
1592  return NULL;
1593 }
struct RIL_Scope * network_entries
Array of networks with global assignment state.
enum GNUNET_NetworkType type
ATS network type.
unsigned int networks_count
Networks count.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Here is the caller graph for this function:

◆ ril_network_is_not_full()

static int ril_network_is_not_full ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  network 
)
static

Determines whether more connections are allocated in a network scope, than they would theoretically fit.

This is used as a heuristic to determine, whether a new connection can be allocated or not.

Parameters
solverthe solver handle
networkthe network scope in question
Returns
GNUNET_YES if there are theoretically enough resources left

Definition at line 1605 of file plugin_ats_ril.c.

References address_count, RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, ril_get_network(), RIL_MIN_BW, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_try_unblock_agent().

1606 {
1607  struct RIL_Scope *net;
1608  struct RIL_Peer_Agent *agent;
1609  unsigned long long address_count = 0;
1610 
1611  for (agent = solver->agents_head; NULL != agent; agent = agent->next)
1612  {
1613  if (agent->address_inuse && agent->is_active)
1614  {
1615  net = agent->address_inuse->solver_information;
1616  if (net->type == network)
1617  {
1618  address_count++;
1619  }
1620  }
1621  }
1622 
1623  net = ril_get_network(solver, network);
1624  return (net->bw_in_available > RIL_MIN_BW * address_count) && (net->bw_out_available > RIL_MIN_BW * address_count);
1625 }
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
static int address_count
Definition: gnunet-hello.c:49
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_try_unblock_agent()

static void ril_try_unblock_agent ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  silent 
)
static

Unblocks an agent for which a connection request is there, that could not be satisfied.

Iterates over the addresses of the agent, if one of its addresses can now be allocated in its scope the agent is unblocked, otherwise it remains unchanged.

Parameters
solverthe solver handle
agentthe agent in question
silent

Definition at line 1638 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, envi_set_active_suggestion(), RIL_Address_Wrapped::next, RIL_MIN_BW, ril_network_is_not_full(), ATS_Address::solver_information, and RIL_Scope::type.

Referenced by GAS_ril_address_add(), GAS_ril_get_preferred_address(), and ril_step().

1639 {
1640  struct RIL_Address_Wrapped *addr_wrap;
1641  struct RIL_Scope *net;
1642  unsigned long long start_in;
1643  unsigned long long start_out;
1644 
1645  for (addr_wrap = agent->addresses_head; NULL != addr_wrap; addr_wrap = addr_wrap->next)
1646  {
1647  net = addr_wrap->address_naked->solver_information;
1648  if (ril_network_is_not_full(solver, net->type))
1649  {
1650  if (NULL == agent->address_inuse)
1651  {
1652  start_in = net->bw_in_available < net->bw_in_utilized ? (net->bw_in_available - net->bw_in_utilized) / 2 : RIL_MIN_BW;
1653  start_out = net->bw_out_available < net->bw_out_utilized ? (net->bw_out_available - net->bw_out_utilized) / 2 : RIL_MIN_BW;
1654  envi_set_active_suggestion(solver, agent, addr_wrap->address_naked, start_in, start_out, silent);
1655  }
1656  return;
1657  }
1658  }
1659  agent->address_inuse = NULL;
1660 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
static int ril_network_is_not_full(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determines whether more connections are allocated in a network scope, than they would theoretically f...
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
static int silent
–silent option
Definition: gnunet-qr.c:49
enum GNUNET_NetworkType type
ATS network type.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_calculate_discount()

static void ril_calculate_discount ( struct GAS_RIL_Handle solver)
static

Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.

Parameters
solverthe solver handle

Definition at line 1669 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::beta, RIL_Learning_Parameters::gamma, GAS_RIL_Handle::global_discount_integrated, GAS_RIL_Handle::global_discount_variable, GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_difference(), GAS_RIL_Handle::parameters, GNUNET_TIME_Relative::rel_value_us, GAS_RIL_Handle::simulate, GAS_RIL_Handle::step_time_last, and RIL_Learning_Parameters::step_time_min.

Referenced by ril_step().

1670 {
1671  struct GNUNET_TIME_Absolute time_now;
1672  struct GNUNET_TIME_Relative time_delta;
1673  double tau;
1674 
1675  // MDP case only for debugging purposes
1676  if (solver->simulate)
1677  {
1678  solver->global_discount_variable = solver->parameters.gamma;
1679  solver->global_discount_integrated = 1;
1680  return;
1681  }
1682 
1683  // semi-MDP case
1684 
1685  //calculate tau, i.e. how many real valued time units have passed, one time unit is one minimum time step
1686  time_now = GNUNET_TIME_absolute_get();
1687  time_delta = GNUNET_TIME_absolute_get_difference(solver->step_time_last, time_now);
1688  solver->step_time_last = time_now;
1689  tau = (double)time_delta.rel_value_us
1690  / (double)solver->parameters.step_time_min.rel_value_us;
1691 
1692  //calculate reward discounts (once per step for all agents)
1693  solver->global_discount_variable = pow(M_E, ((-1.0) * ((double)solver->parameters.beta) * tau));
1694  solver->global_discount_integrated = (1.0 - solver->global_discount_variable)
1695  / (double)solver->parameters.beta;
1696 }
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
uint64_t rel_value_us
The actual value.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
unsigned long long simulate
Simulate steps, i.e.
double beta
Learning discount variable in the TD-update for semi-MDPs.
struct RIL_Learning_Parameters parameters
Learning parameters.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
double global_discount_variable
Variable discount factor, dependent on time between steps.
Time for absolute times used by GNUnet, in microseconds.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_difference(struct GNUNET_TIME_Absolute start, struct GNUNET_TIME_Absolute end)
Compute the time difference between the given start and end times.
Definition: time.c:353
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
Time for relative time used by GNUnet, in microseconds.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_network_count_active_agents()

static int ril_network_count_active_agents ( struct GAS_RIL_Handle solver,
struct RIL_Scope scope 
)
static

Count the number of active agents/connections in a network scope.

Parameters
solverthe solver handle
scopethe network scope in question
Returns
the number of allocated connections

Definition at line 1706 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, and ATS_Address::solver_information.

Referenced by ril_networks_update_state().

1707 {
1708  int c = 0;
1709  struct RIL_Peer_Agent *cur_agent;
1710 
1711  for (cur_agent = solver->agents_head; NULL != cur_agent; cur_agent = cur_agent->next)
1712  {
1713  if (cur_agent->is_active && cur_agent->address_inuse && (cur_agent->address_inuse->solver_information == scope))
1714  {
1715  c++;
1716  }
1717  }
1718  return c;
1719 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the caller graph for this function:

◆ ril_network_get_assigned()

static unsigned long long ril_network_get_assigned ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  type,
int  direction_in 
)
static

Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the outbound direction.

Parameters
solverthe solver handle
typethe type of the network scope in question
direction_inGNUNET_YES if the inbound direction should be summed up, otherwise the outbound direction will be summed up
Returns
the sum of the assigned bandwidths

Definition at line 1732 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_networks_update_state().

1733 {
1734  struct RIL_Peer_Agent *cur;
1735  struct RIL_Scope *net;
1736  unsigned long long sum = 0;
1737 
1738  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1739  {
1740  if (cur->is_active && cur->address_inuse)
1741  {
1742  net = cur->address_inuse->solver_information;
1743  if (net->type == type)
1744  {
1745  if (direction_in)
1746  sum += cur->bw_in;
1747  else
1748  sum += cur->bw_out;
1749  }
1750  }
1751  }
1752 
1753  return sum;
1754 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the caller graph for this function:

◆ ril_network_get_utilized()

static unsigned long long ril_network_get_utilized ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  type,
int  direction_in 
)
static

Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound or in the outbound direction.

Parameters
solverthe solver handle
typethe type of the network scope in question
direction_inGNUNET_YES if the inbound direction should be summed up, otherwise the outbound direction will be summed up
Returns
the sum of the utilized bandwidths (in bytes/second)

Definition at line 1767 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, GAS_NormalizationInfo::norm, ATS_Address::norm_utilization_in, ATS_Address::norm_utilization_out, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_networks_update_state().

1768 {
1769  struct RIL_Peer_Agent *cur;
1770  struct RIL_Scope *net;
1771  unsigned long long sum = 0;
1772 
1773  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1774  {
1775  if (cur->is_active && cur->address_inuse)
1776  {
1777  net = cur->address_inuse->solver_information;
1778  if (net->type == type)
1779  {
1780  if (direction_in)
1781  sum += cur->address_inuse->norm_utilization_in.norm;
1782  else
1784  }
1785  }
1786  }
1787 
1788  return sum;
1789 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
struct GAS_NormalizationInfo norm_utilization_out
Normalized utilization outbound for this address.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
struct GAS_NormalizationInfo norm_utilization_in
Normalized utilization inbound for this address.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
double norm
Normalized values from queue to a range of values [1.0...2.0].
Here is the caller graph for this function:

◆ ril_networks_update_state()

static void ril_networks_update_state ( struct GAS_RIL_Handle solver)
static

Retrieves the state of the network scope, so that its attributes are up-to- date.

Parameters
solverthe solver handle

Definition at line 1798 of file plugin_ats_ril.c.

References RIL_Scope::active_agent_count, RIL_Scope::bw_in_assigned, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_assigned, RIL_Scope::bw_out_utilized, GNUNET_NO, GNUNET_YES, GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, ril_network_count_active_agents(), ril_network_get_assigned(), ril_network_get_social_welfare(), ril_network_get_utilized(), RIL_Scope::social_welfare, and RIL_Scope::type.

Referenced by ril_step().

1799 {
1800  int c;
1801  struct RIL_Scope *net;
1802 
1803  for (c = 0; c < solver->networks_count; c++)
1804  {
1805  net = &solver->network_entries[c];
1811  net->social_welfare = ril_network_get_social_welfare(solver, net);
1812  }
1813 }
#define GNUNET_NO
Definition: gnunet_common.h:78
struct RIL_Scope * network_entries
Array of networks with global assignment state.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
enum GNUNET_NetworkType type
ATS network type.
static unsigned long long ril_network_get_utilized(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound o...
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
double social_welfare
The social welfare achieved in the scope.
static int ril_network_count_active_agents(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Count the number of active agents/connections in a network scope.
unsigned int networks_count
Networks count.
static unsigned long long ril_network_get_assigned(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the ...
unsigned int active_agent_count
Number of active agents in scope.
uint32_t bw_in_assigned
Bandwidth inbound assigned in network after last step.
unsigned long long bw_out_assigned
Bandwidth outbound assigned in network after last step.
#define GNUNET_YES
Definition: gnunet_common.h:77
static double ril_network_get_social_welfare(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Calculates the social welfare within a network scope according to what social welfare measure is set ...
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step_schedule_next()

static void ril_step_schedule_next ( struct GAS_RIL_Handle solver)
static

Schedules the next global step in an adaptive way.

The more resources are left, the earlier the next step is scheduled. This serves the reactivity of the solver to changed inputs.

Parameters
solverthe solver handle

Definition at line 1823 of file plugin_ats_ril.c.

References GAS_RIL_Handle::done, GNUNET_assert, GNUNET_NO, GNUNET_SCHEDULER_add_delayed(), GNUNET_TIME_relative_saturating_multiply(), GNUNET_TIME_relative_subtract(), GNUNET_TIME_UNIT_MICROSECONDS, GNUNET_TIME_UNIT_ZERO, GAS_RIL_Handle::parameters, GNUNET_TIME_Relative::rel_value_us, ril_get_used_resource_ratio(), RIL_INTERVAL_EXPONENT, ril_step_scheduler_task(), GAS_RIL_Handle::simulate, GAS_RIL_Handle::step_next_task_id, RIL_Learning_Parameters::step_time_max, and RIL_Learning_Parameters::step_time_min.

Referenced by ril_step().

1824 {
1825  double used_ratio;
1826  double factor;
1827  double y;
1828  double offset;
1829  struct GNUNET_TIME_Relative time_next;
1830 
1831  used_ratio = ril_get_used_resource_ratio(solver);
1832 
1833  GNUNET_assert(
1836 
1837  factor = (double)GNUNET_TIME_relative_subtract(solver->parameters.step_time_max,
1839  offset = (double)solver->parameters.step_time_min.rel_value_us;
1840  y = factor * pow(used_ratio, RIL_INTERVAL_EXPONENT) + offset;
1841 
1842  GNUNET_assert(y <= (double)solver->parameters.step_time_max.rel_value_us);
1843  GNUNET_assert(y >= (double)solver->parameters.step_time_min.rel_value_us);
1844 
1845  time_next = GNUNET_TIME_relative_saturating_multiply(GNUNET_TIME_UNIT_MICROSECONDS, (unsigned long long)y);
1846 
1847 // LOG (GNUNET_ERROR_TYPE_INFO, "ratio: %f, factor: %f, offset: %f, y: %f\n",
1848 // used_ratio,
1849 // factor,
1850 // offset,
1851 // y);
1852 
1853  if (solver->simulate)
1854  {
1855  time_next = GNUNET_TIME_UNIT_ZERO;
1856  }
1857 
1858  if ((NULL == solver->step_next_task_id) && (GNUNET_NO == solver->done))
1859  {
1861  solver);
1862  }
1863 }
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
uint64_t rel_value_us
The actual value.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define RIL_INTERVAL_EXPONENT
static void ril_step_scheduler_task(void *cls)
Task for the scheduler, which performs one step and lets the solver know that no further step is sche...
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1237
unsigned long long simulate
Simulate steps, i.e.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_subtract(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Subtract relative timestamp from the other.
Definition: time.c:602
struct RIL_Learning_Parameters parameters
Learning parameters.
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
int done
Shutdown.
static double ril_get_used_resource_ratio(struct GAS_RIL_Handle *solver)
Determines how much of the available bandwidth is assigned.
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
#define GNUNET_TIME_UNIT_MICROSECONDS
One microsecond, our basic time unit.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:499
Time for relative time used by GNUnet, in microseconds.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_w_init()

static void agent_w_init ( struct RIL_Peer_Agent agent)
static

Initializes the matrix W of parameter vectors theta with small random numbers.

Parameters
agentThe respective agent

Definition at line 1935 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::alpha, RIL_Peer_Agent::envi, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, and RIL_Peer_Agent::W.

Referenced by agent_init().

1936 {
1937  int i;
1938  int k;
1939 
1940  for (i = 0; i < agent->n; i++)
1941  {
1942  for (k = 0; k < agent->m; k++)
1943  {
1944  agent->W[i][k] = agent->envi->parameters.alpha * (1.0 - 2.0 * ((double)GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX) / (double)UINT32_MAX));
1945  }
1946  }
1947 }
unsigned int m
Number of rows of W / Number of state-vector features.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
unsigned int n
Number of columns of W / Number of actions.
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
double alpha
Gradient-descent step-size.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_init()

static struct RIL_Peer_Agent* agent_init ( void *  s,
const struct GNUNET_PeerIdentity peer 
)
static

Initialize an agent without addresses and its knowledge base.

Parameters
sril solver
peerthe one in question
Returns
handle to the new agent

Definition at line 1957 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, agent_w_init(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::E, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, GNUNET_malloc, GNUNET_new, GNUNET_NO, RIL_Peer_Agent::is_active, RIL_Peer_Agent::m, RIL_Peer_Agent::n, RIL_Peer_Agent::nop_bonus, RIL_Peer_Agent::objective_old, peer, RIL_Peer_Agent::peer, RIL_ACTION_INVALID, RIL_ACTION_TYPE_NUM, RIL_MIN_BW, RIL_Peer_Agent::s_old, RIL_Peer_Agent::step_count, RIL_Peer_Agent::suggestion_issue, and RIL_Peer_Agent::W.

Referenced by ril_get_agent().

1958 {
1959  int i;
1960  struct GAS_RIL_Handle * solver = s;
1961  struct RIL_Peer_Agent * agent = GNUNET_new(struct RIL_Peer_Agent);
1962 
1963  agent->envi = solver;
1964  agent->peer = *peer;
1965  agent->step_count = 0;
1966  agent->is_active = GNUNET_NO;
1967  agent->bw_in = RIL_MIN_BW;
1968  agent->bw_out = RIL_MIN_BW;
1969  agent->suggestion_issue = GNUNET_NO;
1970  agent->n = RIL_ACTION_TYPE_NUM;
1971  agent->m = 0;
1972  agent->W = (double **)GNUNET_malloc(sizeof(double *) * agent->n);
1973  agent->E = (double **)GNUNET_malloc(sizeof(double *) * agent->n);
1974  for (i = 0; i < agent->n; i++)
1975  {
1976  agent->W[i] = (double *)GNUNET_malloc(sizeof(double) * agent->m);
1977  agent->E[i] = (double *)GNUNET_malloc(sizeof(double) * agent->m);
1978  }
1979  agent_w_init(agent);
1980  agent->eligibility_reset = GNUNET_NO;
1981  agent->a_old = RIL_ACTION_INVALID;
1982  agent->s_old = GNUNET_malloc(sizeof(double) * agent->m);
1983  agent->address_inuse = NULL;
1984  agent->objective_old = 0;
1985  agent->nop_bonus = 0;
1986 
1987  return agent;
1988 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
unsigned int m
Number of rows of W / Number of state-vector features.
A handle for the reinforcement learning solver.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
int a_old
Last chosen action.
#define GNUNET_NO
Definition: gnunet_common.h:78
int suggestion_issue
Flag whether a suggestion has to be issued.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
struct GNUNET_PeerIdentity peer
Peer ID.
static void agent_w_init(struct RIL_Peer_Agent *agent)
Initializes the matrix W of parameter vectors theta with small random numbers.
double ** W
Experience matrix W.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.
double ** E
Eligibility traces.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
double * s_old
Last perceived state feature vector.
struct GAS_RIL_Handle * envi
Environment handle.
#define GNUNET_malloc(size)
Wrapper around malloc.
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
unsigned long long step_count
Number of performed time-steps.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_die()

static void agent_die ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Deallocate agent.

Parameters
solverthe solver handle
agentthe agent to retire

Definition at line 1997 of file plugin_ats_ril.c.

References RIL_Peer_Agent::E, GNUNET_free, GNUNET_free_non_null, RIL_Peer_Agent::n, RIL_Peer_Agent::s_old, and RIL_Peer_Agent::W.

Referenced by libgnunet_plugin_ats_ril_done().

1998 {
1999  int i;
2000 
2001  for (i = 0; i < agent->n; i++)
2002  {
2003  GNUNET_free_non_null(agent->W[i]);
2004  GNUNET_free_non_null(agent->E[i]);
2005  }
2006  GNUNET_free_non_null(agent->W);
2007  GNUNET_free_non_null(agent->E);
2008  GNUNET_free_non_null(agent->s_old);
2009  GNUNET_free(agent);
2010 }
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
double ** W
Experience matrix W.
double ** E
Eligibility traces.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ ril_get_agent()

static struct RIL_Peer_Agent* ril_get_agent ( struct GAS_RIL_Handle solver,
const struct GNUNET_PeerIdentity peer,
int  create 
)
static

Returns the agent for a peer.

Parameters
solverthe solver handle
peerthe identity of the peer
createwhether or not to create an agent, if none is allocated yet
Returns
the agent

Definition at line 2021 of file plugin_ats_ril.c.

References agent_init(), GAS_RIL_Handle::agents_head, GAS_RIL_Handle::agents_tail, GNUNET_CONTAINER_DLL_insert_tail, GNUNET_memcmp, RIL_Peer_Agent::next, and RIL_Peer_Agent::peer.

Referenced by GAS_ril_address_add(), GAS_ril_address_delete(), GAS_ril_get_preferred_address(), and GAS_ril_stop_get_preferred_address().

2022 {
2023  struct RIL_Peer_Agent *cur;
2024 
2025  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
2026  {
2027  if (0 == GNUNET_memcmp(peer, &cur->peer))
2028  {
2029  return cur;
2030  }
2031  }
2032 
2033  if (create)
2034  {
2035  cur = agent_init(solver, peer);
2037  return cur;
2038  }
2039  return NULL;
2040 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
struct RIL_Peer_Agent * agents_tail
struct GNUNET_PeerIdentity peer
Peer ID.
static struct RIL_Peer_Agent * agent_init(void *s, const struct GNUNET_PeerIdentity *peer)
Initialize an agent without addresses and its knowledge base.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
#define GNUNET_memcmp(a, b)
Compare memory in a and b, where both must be of the same pointer type.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_network_is_active()

static int ril_network_is_active ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  network 
)
static

Determine whether at least the minimum bandwidth is set for the network.

Otherwise the network is considered inactive and not used. Addresses in an inactive network are ignored.

Parameters
solversolver handle
networkthe network type
Returns
whether or not the network is considered active

Definition at line 2051 of file plugin_ats_ril.c.

References RIL_Scope::bw_out_available, ril_get_network(), and RIL_MIN_BW.

Referenced by GAS_ril_address_add(), and GAS_ril_address_delete().

2052 {
2053  struct RIL_Scope *net;
2054 
2055  net = ril_get_network(solver, network);
2056  return net->bw_out_available >= RIL_MIN_BW;
2057 }
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
uint32_t bw_out_available
Total available outbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_cut_from_vector()

static void ril_cut_from_vector ( void **  old,
size_t  element_size,
unsigned int  hole_start,
unsigned int  hole_length,
unsigned int  old_length 
)
static

Cuts a slice out of a vector of elements.

This is used to decrease the size of the matrix storing the reward function approximation. It copies the memory, which is not cut, to the new vector, frees the memory of the old vector, and redirects the pointer to the new one.

Parameters
oldpointer to the pointer to the first element of the vector
element_sizebyte size of the vector elements
hole_startthe first element to cut out
hole_lengththe number of elements to cut out
old_lengththe length of the old vector

Definition at line 2071 of file plugin_ats_ril.c.

References GNUNET_assert, GNUNET_free, GNUNET_malloc, GNUNET_memcpy, and size.

Referenced by GAS_ril_address_delete().

2076 {
2077  char *tmpptr;
2078  char *oldptr = (char *)*old;
2079  size_t size;
2080  unsigned int bytes_before;
2081  unsigned int bytes_hole;
2082  unsigned int bytes_after;
2083 
2084  GNUNET_assert(old_length >= hole_length);
2085  GNUNET_assert(old_length >= (hole_start + hole_length));
2086 
2087  size = element_size * (old_length - hole_length);
2088 
2089  bytes_before = element_size * hole_start;
2090  bytes_hole = element_size * hole_length;
2091  bytes_after = element_size * (old_length - hole_start - hole_length);
2092 
2093  if (0 == size)
2094  {
2095  tmpptr = NULL;
2096  }
2097  else
2098  {
2099  tmpptr = GNUNET_malloc(size);
2100  GNUNET_memcpy(tmpptr, oldptr, bytes_before);
2101  GNUNET_memcpy(tmpptr + bytes_before, oldptr + (bytes_before + bytes_hole), bytes_after);
2102  }
2103  if (NULL != *old)
2104  {
2105  GNUNET_free(*old);
2106  }
2107  *old = (void *)tmpptr;
2108 }
static unsigned int element_size
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
static unsigned int size
Size of the "table".
Definition: peer.c:66
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ GAS_ril_address_change_preference()

static void GAS_ril_address_change_preference ( void *  solver,
const struct GNUNET_PeerIdentity peer,
enum GNUNET_ATS_PreferenceKind  kind,
double  pref_rel 
)
static

Change relative preference for quality in solver.

Parameters
solverthe solver handle
peerthe peer to change the preference for
kindthe kind to change the preference
pref_relthe normalized preference value for this kind over all clients

Definition at line 2124 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ATS_print_preference_type(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, GAS_RIL_Handle::parameters, ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2128 {
2130  "API_address_change_preference() Preference '%s' for peer '%s' changed to %.2f \n",
2131  GNUNET_ATS_print_preference_type(kind), GNUNET_i2s(peer), pref_rel);
2132 
2133  struct GAS_RIL_Handle *s = solver;
2134 
2137  ril_step(s);
2138 }
A handle for the reinforcement learning solver.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
double temperature
Softmax action-selection temperature.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_init
Initial exploration ratio value.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_add()

static void GAS_ril_address_add ( void *  solver,
struct ATS_Address address,
uint32_t  network 
)
static

Add a new address for a peer to the solver.

The address is already contained in the addresses hashmap!

Parameters
solverthe solver Handle
addressthe address to add
networknetwork type of this address

Definition at line 2151 of file plugin_ats_ril.c.

References ATS_Address::active, ATS_Address::addr, address, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::addresses_tail, RIL_Peer_Agent::E, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_array_grow, GNUNET_CONTAINER_DLL_insert_tail, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_new, GNUNET_NO, GNUNET_YES, LOG, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, ATS_Address::peer, ATS_Address::plugin, RIL_Learning_Parameters::rbf_divisor, ril_get_agent(), ril_get_network(), ril_network_is_active(), ril_step(), ril_try_unblock_agent(), RIL_Peer_Agent::s_old, ATS_Address::solver_information, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_init, RIL_Peer_Agent::W, and zero.

Referenced by libgnunet_plugin_ats_ril_init().

2154 {
2155  struct GAS_RIL_Handle *s = solver;
2156  struct RIL_Peer_Agent *agent;
2157  struct RIL_Address_Wrapped *address_wrapped;
2158  struct RIL_Scope *net;
2159  unsigned int m_new;
2160  unsigned int m_old;
2161  unsigned int n_new;
2162  unsigned int n_old;
2163  int i;
2164  unsigned int zero;
2165 
2167  "API_address_add()\n");
2168 
2169  net = ril_get_network(s, network);
2170  address->solver_information = net;
2171 
2172  if (!ril_network_is_active(s, network))
2173  {
2175  "API_address_add() Did not add %s address %s for peer '%s', network does not have enough bandwidth\n",
2176  address->plugin, address->addr, GNUNET_i2s(&address->peer));
2177  return;
2178  }
2179 
2182 
2183  agent = ril_get_agent(s, &address->peer, GNUNET_YES);
2184 
2185  //add address
2186  address_wrapped = GNUNET_new(struct RIL_Address_Wrapped);
2187  address_wrapped->address_naked = address;
2188  GNUNET_CONTAINER_DLL_insert_tail(agent->addresses_head, agent->addresses_tail, address_wrapped);
2189 
2190  //increase size of W
2191  m_new = agent->m + ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1));
2192  m_old = agent->m;
2193  n_new = agent->n + 1;
2194  n_old = agent->n;
2195 
2196  GNUNET_array_grow(agent->W, agent->n, n_new);
2197  agent->n = n_old;
2198  GNUNET_array_grow(agent->E, agent->n, n_new);
2199  for (i = 0; i < n_new; i++)
2200  {
2201  if (i < n_old)
2202  {
2203  agent->m = m_old;
2204  GNUNET_array_grow(agent->W[i], agent->m, m_new);
2205  agent->m = m_old;
2206  GNUNET_array_grow(agent->E[i], agent->m, m_new);
2207  }
2208  else
2209  {
2210  zero = 0;
2211  GNUNET_array_grow(agent->W[i], zero, m_new);
2212  zero = 0;
2213  GNUNET_array_grow(agent->E[i], zero, m_new);
2214  }
2215  }
2216 
2217  //increase size of old state vector
2218  agent->m = m_old;
2219  GNUNET_array_grow(agent->s_old, agent->m, m_new);
2220 
2221  ril_try_unblock_agent(s, agent, GNUNET_NO);
2222 
2223  ril_step(s);
2224 
2225  LOG(GNUNET_ERROR_TYPE_DEBUG, "API_address_add() Added %s %s address %s for peer '%s'\n",
2226  address->active ? "active" : "inactive", address->plugin, address->addr,
2227  GNUNET_i2s(&address->peer));
2228 }
static const struct GNUNET_CRYPTO_EcdsaPrivateKey zero
Public key of all zeros.
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
int active
Is this the active address for this peer?
unsigned int m
Number of rows of W / Number of state-vector features.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
#define GNUNET_NO
Definition: gnunet_common.h:78
char * plugin
Plugin name.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
struct ATS_Address * address_naked
The address.
double temperature
Softmax action-selection temperature.
#define GNUNET_array_grow(arr, size, tsize)
Grow a well-typed (!) array.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
double epsilon_init
Initial exploration ratio value.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
unsigned long long rbf_divisor
State space divisor.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
static char * address
GNS address for this phone.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Wrapper for addresses to store them in agent&#39;s linked list.
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_delete()

static void GAS_ril_address_delete ( void *  solver,
struct ATS_Address address 
)
static

Delete an address in the solver.

The address is not contained in the address hashmap anymore!

Parameters
solverthe solver handle
addressthe address to remove

Definition at line 2239 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, ATS_Address::active, ATS_Address::addr, RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), agent_address_get_wrapped(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::E, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_assert, GNUNET_CONTAINER_DLL_remove, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_free_non_null, GNUNET_i2s(), GNUNET_log, GNUNET_NO, GNUNET_YES, LOG, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, ATS_Address::peer, ATS_Address::plugin, RIL_Learning_Parameters::rbf_divisor, RIL_ACTION_INVALID, RIL_ACTION_TYPE_NUM, ril_cut_from_vector(), ril_get_agent(), ril_network_is_active(), ril_step(), RIL_Peer_Agent::s_old, ATS_Address::solver_information, RIL_Peer_Agent::suggestion_address, RIL_Peer_Agent::suggestion_issue, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_init, RIL_Scope::type, and RIL_Peer_Agent::W.

Referenced by libgnunet_plugin_ats_ril_init().

2241 {
2242  struct GAS_RIL_Handle *s = solver;
2243  struct RIL_Peer_Agent *agent;
2244  struct RIL_Address_Wrapped *address_wrapped;
2245  int address_index;
2246  unsigned int m_new;
2247  unsigned int n_new;
2248  int i;
2249  struct RIL_Scope *net;
2250 
2252  "API_address_delete() Delete %s %s address %s for peer '%s'\n",
2253  address->active ? "active" : "inactive",
2254  address->plugin,
2255  address->addr,
2256  GNUNET_i2s(&address->peer));
2257 
2258  agent = ril_get_agent(s, &address->peer, GNUNET_NO);
2259  if (NULL == agent)
2260  {
2261  net = address->solver_information;
2264  "No agent allocated for peer yet, since address was in inactive network\n");
2265  return;
2266  }
2267 
2270 
2271  address_index = agent_address_get_index(agent, address);
2272  address_wrapped = agent_address_get_wrapped(agent, address);
2273 
2274  if (NULL == address_wrapped)
2275  {
2276  net = address->solver_information;
2278  "Address not considered by agent, address was in inactive network\n");
2279  return;
2280  }
2282  agent->addresses_tail,
2283  address_wrapped);
2284  GNUNET_free(address_wrapped);
2285 
2286  //decrease W
2287  m_new = agent->m - ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1));
2288  n_new = agent->n - 1;
2289 
2290  for (i = 0; i < agent->n; i++)
2291  {
2292  ril_cut_from_vector((void **)&agent->W[i], sizeof(double),
2293  address_index * ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1)),
2294  ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1)), agent->m);
2295  ril_cut_from_vector((void **)&agent->E[i], sizeof(double),
2296  address_index * ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1)),
2297  ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1)), agent->m);
2298  }
2299  GNUNET_free_non_null(agent->W[RIL_ACTION_TYPE_NUM + address_index]);
2300  GNUNET_free_non_null(agent->E[RIL_ACTION_TYPE_NUM + address_index]);
2301  ril_cut_from_vector((void **)&agent->W, sizeof(double *), RIL_ACTION_TYPE_NUM + address_index,
2302  1, agent->n);
2303  ril_cut_from_vector((void **)&agent->E, sizeof(double *), RIL_ACTION_TYPE_NUM + address_index,
2304  1, agent->n);
2305  //correct last action
2306  if (agent->a_old > (RIL_ACTION_TYPE_NUM + address_index))
2307  {
2308  agent->a_old -= 1;
2309  }
2310  else if (agent->a_old == (RIL_ACTION_TYPE_NUM + address_index))
2311  {
2312  agent->a_old = RIL_ACTION_INVALID;
2313  }
2314  //decrease old state vector
2315  ril_cut_from_vector((void **)&agent->s_old, sizeof(double),
2316  address_index * ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1)),
2317  ((s->parameters.rbf_divisor + 1) * (s->parameters.rbf_divisor + 1)), agent->m);
2318  agent->m = m_new;
2319  agent->n = n_new;
2320 
2321  if (agent->address_inuse == address)
2322  {
2323  if (NULL != agent->addresses_head) //if peer has an address left, use it
2324  {
2326  "Active address died, suggesting alternative!\n");
2328  agent,
2329  agent->addresses_head->address_naked,
2330  agent->bw_in,
2331  agent->bw_out,
2332  GNUNET_YES);
2333  }
2334  else
2335  {
2337  "Active address died, suggesting disconnect!\n");
2338  envi_set_active_suggestion(s, agent, NULL, 0, 0, GNUNET_NO);
2339  }
2340  }
2341  ril_step(solver);
2342  if (agent->suggestion_address == address)
2343  {
2344  agent->suggestion_issue = GNUNET_NO;
2345  agent->suggestion_address = NULL;
2346  }
2347  GNUNET_assert(agent->address_inuse != address);
2348 }
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int active
Is this the active address for this peer?
unsigned int m
Number of rows of W / Number of state-vector features.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
int a_old
Last chosen action.
static void ril_cut_from_vector(void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
Cuts a slice out of a vector of elements.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
char * plugin
Plugin name.
int suggestion_issue
Flag whether a suggestion has to be issued.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
double temperature
Softmax action-selection temperature.
enum GNUNET_NetworkType type
ATS network type.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
static struct RIL_Address_Wrapped * agent_address_get_wrapped(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Gets the wrapped address from the agent&#39;s list.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_log(kind,...)
double * s_old
Last perceived state feature vector.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
unsigned long long rbf_divisor
State space divisor.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
struct ATS_Address * suggestion_address
The address which has to be issued.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Wrapper for addresses to store them in agent&#39;s linked list.
double temperature_init
Initial softmax action-selection temperature.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_property_changed()

static void GAS_ril_address_property_changed ( void *  solver,
struct ATS_Address address 
)
static

Update the properties of an address in the solver.

Parameters
solversolver handle
addressthe address

Definition at line 2358 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, GAS_RIL_Handle::parameters, ATS_Address::peer, ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2360 {
2361  struct GAS_RIL_Handle *s = solver;
2362 
2364  "Properties for peer '%s' address changed\n",
2365  GNUNET_i2s(&address->peer));
2368  ril_step(s);
2369 }
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
A handle for the reinforcement learning solver.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_init
Initial exploration ratio value.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_preference_feedback()

static void GAS_ril_address_preference_feedback ( void *  solver,
struct GNUNET_SERVICE_Client application,
const struct GNUNET_PeerIdentity peer,
const struct GNUNET_TIME_Relative  scope,
enum GNUNET_ATS_PreferenceKind  kind,
double  score 
)
static

Give feedback about the current assignment.

Parameters
solverthe solver handle
applicationthe application
peerthe peer to change the preference for
scopethe time interval for this feedback: [now - scope .. now]
kindthe kind to change the preference
scorethe score

Definition at line 2383 of file plugin_ats_ril.c.

References GNUNET_ATS_print_preference_type(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, and GNUNET_TIME_Relative::rel_value_us.

Referenced by libgnunet_plugin_ats_ril_init().

2389 {
2391  "API_address_preference_feedback() Peer '%s' got a feedback of %+.3f from application %s for "
2392  "preference %s for %d seconds\n",
2393  GNUNET_i2s(peer),
2394  "UNKNOWN",
2396  scope.rel_value_us / 1000000);
2397 }
uint64_t rel_value_us
The actual value.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
#define LOG(kind,...)
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_bulk_start()

static void GAS_ril_bulk_start ( void *  solver)
static

Start a bulk operation.

Parameters
solverthe solver

Definition at line 2406 of file plugin_ats_ril.c.

References GAS_RIL_Handle::bulk_lock, GNUNET_ERROR_TYPE_DEBUG, and LOG.

Referenced by libgnunet_plugin_ats_ril_init().

2407 {
2408  struct GAS_RIL_Handle *s = solver;
2409 
2411  "API_bulk_start() lock: %d\n", s->bulk_lock + 1);
2412 
2413  s->bulk_lock++;
2414 }
A handle for the reinforcement learning solver.
int bulk_lock
Lock for bulk operations.
#define LOG(kind,...)
Here is the caller graph for this function:

◆ GAS_ril_bulk_stop()

static void GAS_ril_bulk_stop ( void *  solver)
static

Bulk operation done.

Parameters
solverthe solver handle

Definition at line 2423 of file plugin_ats_ril.c.

References GAS_RIL_Handle::bulk_changes, GAS_RIL_Handle::bulk_lock, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, LOG, and ril_step().

Referenced by libgnunet_plugin_ats_ril_init().

2424 {
2425  struct GAS_RIL_Handle *s = solver;
2426 
2428  "API_bulk_stop() lock: %d\n",
2429  s->bulk_lock - 1);
2430 
2431  if (s->bulk_lock < 1)
2432  {
2433  GNUNET_break(0);
2434  return;
2435  }
2436  s->bulk_lock--;
2437 
2438  if (0 < s->bulk_changes)
2439  {
2440  ril_step(solver);
2441  s->bulk_changes = 0;
2442  }
2443 }
int bulk_changes
Number of changes during a lock.
A handle for the reinforcement learning solver.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
int bulk_lock
Lock for bulk operations.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_get_preferred_address()

static void GAS_ril_get_preferred_address ( void *  solver,
const struct GNUNET_PeerIdentity peer 
)
static

Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth changed callback.

The solver must only notify about changes for peers with pending address requests!

Parameters
solverthe solver handle
peerthe identity of the peer

Definition at line 2457 of file plugin_ats_ril.c.

References ATS_Address::addr, RIL_Peer_Agent::address_inuse, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_YES, RIL_Peer_Agent::is_active, LOG, GAS_RIL_Handle::parameters, ATS_Address::plugin, ril_get_agent(), ril_try_unblock_agent(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2459 {
2460  struct GAS_RIL_Handle *s = solver;
2461  struct RIL_Peer_Agent *agent;
2462 
2463  LOG(GNUNET_ERROR_TYPE_DEBUG, "API_get_preferred_address()\n");
2464 
2465  agent = ril_get_agent(s, peer, GNUNET_YES);
2466 
2467  agent->is_active = GNUNET_YES;
2468  envi_set_active_suggestion(solver, agent, agent->address_inuse, agent->bw_in, agent->bw_out, GNUNET_YES);
2469 
2470  ril_try_unblock_agent(solver, agent, GNUNET_YES);
2471 
2472  if (agent->address_inuse)
2473  {
2475  "API_get_preferred_address() Activated agent for peer '%s' with %s address %s\n",
2476  GNUNET_i2s(peer), agent->address_inuse->plugin, agent->address_inuse->addr);
2477  }
2478  else
2479  {
2481  "API_get_preferred_address() Activated agent for peer '%s', but no address available\n",
2482  GNUNET_i2s(peer));
2485  }
2486  if (NULL != agent->address_inuse)
2487  s->env->bandwidth_changed_cb(s->env->cls,
2488  agent->address_inuse);
2489 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
char * plugin
Plugin name.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:77
#define LOG(kind,...)
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_stop_get_preferred_address()

static void GAS_ril_stop_get_preferred_address ( void *  solver,
const struct GNUNET_PeerIdentity peer 
)
static

Tell solver stop notifying ATS about changes for this peers.

The solver must only notify about changes for peers with pending address requests!

Parameters
solverthe solver handle
peerthe peer

Definition at line 2502 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::is_active, LOG, GAS_RIL_Handle::parameters, ril_get_agent(), ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2504 {
2505  struct GAS_RIL_Handle *s = solver;
2506  struct RIL_Peer_Agent *agent;
2507 
2509  "API_stop_get_preferred_address()");
2510 
2511  agent = ril_get_agent(s, peer, GNUNET_NO);
2512 
2513  if (NULL == agent)
2514  {
2515  GNUNET_break(0);
2516  return;
2517  }
2518  if (GNUNET_NO == agent->is_active)
2519  {
2520  GNUNET_break(0);
2521  return;
2522  }
2523 
2526 
2527  agent->is_active = GNUNET_NO;
2528 
2529  envi_set_active_suggestion(s, agent, agent->address_inuse, agent->bw_in, agent->bw_out,
2530  GNUNET_YES);
2531 
2532  ril_step(s);
2533 
2535  "API_stop_get_preferred_address() Paused agent for peer '%s'\n",
2536  GNUNET_i2s(peer));
2537 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
A handle for the reinforcement learning solver.
#define GNUNET_NO
Definition: gnunet_common.h:78
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
#define GNUNET_YES
Definition: gnunet_common.h:77
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ libgnunet_plugin_ats_ril_init()

void* libgnunet_plugin_ats_ril_init ( void *  cls)

Entry point for the plugin.

Parameters
clspointer to the 'struct GNUNET_ATS_PluginEnvironment'

Definition at line 2546 of file plugin_ats_ril.c.

References _, RIL_Learning_Parameters::algorithm, RIL_Learning_Parameters::alpha, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, RIL_Learning_Parameters::beta, RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, GNUNET_ATS_PluginEnvironment::cfg, GNUNET_ATS_SolverFunctions::cls, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::done, RIL_Learning_Parameters::eligibility_trace_mode, env, GAS_RIL_Handle::env, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_decay, RIL_Learning_Parameters::epsilon_init, RIL_Learning_Parameters::gamma, GAS_ril_address_add(), GAS_ril_address_change_preference(), GAS_ril_address_delete(), GAS_ril_address_preference_feedback(), GAS_ril_address_property_changed(), GAS_ril_bulk_start(), GAS_ril_bulk_stop(), GAS_ril_get_preferred_address(), GAS_ril_stop_get_preferred_address(), GNUNET_ATS_PluginEnvironment::get_preferences, GNUNET_assert, GNUNET_CONFIGURATION_get_value_float(), GNUNET_CONFIGURATION_get_value_number(), GNUNET_CONFIGURATION_get_value_string(), GNUNET_CONFIGURATION_get_value_time(), GNUNET_CONFIGURATION_get_value_yesno(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_free, GNUNET_malloc, GNUNET_new, GNUNET_NO, GNUNET_NT_to_string(), GNUNET_OK, GNUNET_STRINGS_utf8_toupper(), GNUNET_SYSERR, GNUNET_YES, GNUNET_ATS_PluginEnvironment::in_quota, RIL_Learning_Parameters::lambda, LOG, GNUNET_ATS_PluginEnvironment::network_count, GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, GNUNET_ATS_PluginEnvironment::out_quota, GAS_RIL_Handle::parameters, RIL_Learning_Parameters::rbf_divisor, RIL_ALGO_Q, RIL_ALGO_SARSA, RIL_DEFAULT_ALGORITHM, RIL_DEFAULT_DISCOUNT_BETA, RIL_DEFAULT_DISCOUNT_GAMMA, RIL_DEFAULT_EXPLORE_DECAY, RIL_DEFAULT_EXPLORE_RATIO, RIL_DEFAULT_GRADIENT_STEP_SIZE, RIL_DEFAULT_RBF_DIVISOR, RIL_DEFAULT_SELECT, RIL_DEFAULT_STEP_TIME_MAX, RIL_DEFAULT_STEP_TIME_MIN, RIL_DEFAULT_TEMPERATURE, RIL_DEFAULT_TEMPERATURE_DECAY, RIL_DEFAULT_TRACE_DECAY, RIL_DEFAULT_WELFARE, RIL_E_ACCUMULATE, RIL_E_REPLACE, RIL_SELECT_EGREEDY, RIL_SELECT_SOFTMAX, RIL_WELFARE_EGALITARIAN, RIL_WELFARE_NASH, GNUNET_ATS_SolverFunctions::s_add, GNUNET_ATS_SolverFunctions::s_address_update_property, GNUNET_ATS_SolverFunctions::s_bulk_start, GNUNET_ATS_SolverFunctions::s_bulk_stop, GNUNET_ATS_SolverFunctions::s_del, GNUNET_ATS_SolverFunctions::s_feedback, GNUNET_ATS_SolverFunctions::s_get, GNUNET_ATS_SolverFunctions::s_get_stop, GNUNET_ATS_SolverFunctions::s_pref, RIL_Learning_Parameters::select, sf, GAS_RIL_Handle::simulate, RIL_Learning_Parameters::social_welfare, GNUNET_ATS_PluginEnvironment::stats, GAS_RIL_Handle::step_count, RIL_Learning_Parameters::step_time_max, RIL_Learning_Parameters::step_time_min, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_decay, RIL_Learning_Parameters::temperature_init, and RIL_Scope::type.

2547 {
2548  static struct GNUNET_ATS_SolverFunctions sf;
2550  struct GAS_RIL_Handle *solver = GNUNET_new(struct GAS_RIL_Handle);
2551  struct RIL_Scope * cur;
2552  int c;
2553  char *string;
2554  float f_tmp;
2555 
2557  "API_init() Initializing RIL solver\n");
2558 
2559  GNUNET_assert(NULL != env);
2560  GNUNET_assert(NULL != env->cfg);
2561  GNUNET_assert(NULL != env->stats);
2562  GNUNET_assert(NULL != env->bandwidth_changed_cb);
2563  GNUNET_assert(NULL != env->get_preferences);
2564 
2565  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number(env->cfg, "ats", "RIL_RBF_DIVISOR", &solver->parameters.rbf_divisor))
2566  {
2568  }
2569 
2570  if (GNUNET_OK
2571  != GNUNET_CONFIGURATION_get_value_time(env->cfg, "ats", "RIL_STEP_TIME_MIN",
2572  &solver->parameters.step_time_min))
2573  {
2575  }
2576 
2577  if (GNUNET_OK
2578  != GNUNET_CONFIGURATION_get_value_time(env->cfg, "ats", "RIL_STEP_TIME_MAX",
2579  &solver->parameters.step_time_max))
2580  {
2582  }
2583 
2584  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(env->cfg, "ats", "RIL_ALGORITHM", &string))
2585  {
2586  GNUNET_STRINGS_utf8_toupper(string, string);
2587  if (0 == strcmp(string, "SARSA"))
2588  {
2590  }
2591  if (0 == strcmp(string, "Q-LEARNING"))
2592  {
2593  solver->parameters.algorithm = RIL_ALGO_Q;
2594  }
2595 
2596  GNUNET_free(string);
2597  }
2598  else
2599  {
2601  }
2602 
2603  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(env->cfg, "ats", "RIL_SELECT", &string))
2604  {
2605  solver->parameters.select = !strcmp(string, "EGREEDY") ? RIL_SELECT_EGREEDY : RIL_SELECT_SOFTMAX;
2606  GNUNET_free(string);
2607  }
2608  else
2609  {
2611  }
2612 
2613 
2616  "RIL_DISCOUNT_BETA", &f_tmp))
2617  {
2618  if (f_tmp < 0.0)
2619  {
2620  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2621  "RIL_DISCOUNT_BETA", f_tmp);
2622  }
2623  else
2624  {
2625  solver->parameters.beta = f_tmp;
2626  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2627  "RIL_DISCOUNT_BETA", f_tmp);
2628  }
2629  }
2630 
2633  "RIL_DISCOUNT_GAMMA", &f_tmp))
2634  {
2635  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2636  {
2637  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2638  "RIL_DISCOUNT_GAMMA", f_tmp);
2639  }
2640  else
2641  {
2642  solver->parameters.gamma = f_tmp;
2643  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2644  "RIL_DISCOUNT_GAMMA", f_tmp);
2645  }
2646  }
2647 
2650  "RIL_GRADIENT_STEP_SIZE", &f_tmp))
2651  {
2652  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2653  {
2654  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2655  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2656  }
2657  else
2658  {
2659  solver->parameters.alpha = f_tmp;
2660  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2661  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2662  }
2663  }
2664 
2667  "RIL_TRACE_DECAY", &f_tmp))
2668  {
2669  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2670  {
2671  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2672  "RIL_TRACE_DECAY", f_tmp);
2673  }
2674  else
2675  {
2676  solver->parameters.lambda = f_tmp;
2677  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2678  "RIL_TRACE_DECAY", f_tmp);
2679  }
2680  }
2681 
2684  "RIL_EXPLORE_RATIO", &f_tmp))
2685  {
2686  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2687  {
2688  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2689  "RIL_EXPLORE_RATIO", f_tmp);
2690  }
2691  else
2692  {
2693  solver->parameters.epsilon_init = f_tmp;
2694  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2695  "RIL_EXPLORE_RATIO", f_tmp);
2696  }
2697  }
2698 
2701  "RIL_EXPLORE_DECAY", &f_tmp))
2702  {
2703  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2704  {
2705  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2706  "RIL_EXPLORE_DECAY", f_tmp);
2707  }
2708  else
2709  {
2710  solver->parameters.epsilon_decay = f_tmp;
2711  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2712  "RIL_EXPLORE_DECAY", f_tmp);
2713  }
2714  }
2715 
2718  "RIL_TEMPERATURE", &f_tmp))
2719  {
2720  if (f_tmp <= 0.0)
2721  {
2722  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2723  "RIL_TEMPERATURE", f_tmp);
2724  }
2725  else
2726  {
2727  solver->parameters.temperature_init = f_tmp;
2728  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2729  "RIL_TEMPERATURE", f_tmp);
2730  }
2731  }
2732 
2735  "RIL_TEMPERATURE_DECAY", &f_tmp))
2736  {
2737  if ((f_tmp <= 0.0) || solver->parameters.temperature_decay > 1)
2738  {
2739  LOG(GNUNET_ERROR_TYPE_ERROR, _("Invalid %s configuration %f \n"),
2740  "RIL_TEMPERATURE_DECAY", f_tmp);
2741  }
2742  else
2743  {
2744  solver->parameters.temperature_decay = f_tmp;
2745  LOG(GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2746  "RIL_TEMPERATURE_DECAY", f_tmp);
2747  }
2748  }
2749 
2750  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number(env->cfg, "ats", "RIL_SIMULATE", &solver->simulate))
2751  {
2752  solver->simulate = GNUNET_NO;
2753  }
2754 
2755  if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(env->cfg, "ats", "RIL_REPLACE_TRACES"))
2756  {
2758  }
2759  else
2760  {
2762  }
2763 
2764  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(env->cfg, "ats", "RIL_SOCIAL_WELFARE", &string))
2765  {
2766  solver->parameters.social_welfare = !strcmp(string, "NASH") ? RIL_WELFARE_NASH : RIL_WELFARE_EGALITARIAN;
2767  GNUNET_free(string);
2768  }
2769  else
2770  {
2772  }
2773 
2774  solver->env = env;
2775  sf.cls = solver;
2785 
2786  solver->networks_count = env->network_count;
2787  solver->network_entries = GNUNET_malloc(env->network_count * sizeof(struct RIL_Scope));
2788  solver->step_count = 0;
2789  solver->done = GNUNET_NO;
2790 
2791  for (c = 0; c < env->network_count; c++)
2792  {
2793  cur = &solver->network_entries[c];
2794  cur->type = c;
2795  cur->bw_in_available = env->in_quota[c];
2796  cur->bw_out_available = env->out_quota[c];
2798  "init() Quotas for %s network: IN %llu - OUT %llu\n",
2799  GNUNET_NT_to_string(cur->type),
2800  cur->bw_in_available / 1024,
2801  cur->bw_out_available / 1024);
2802  }
2803 
2804  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() Parameters:\n");
2805  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() Algorithm = %s, alpha = %f, beta = %f, lambda = %f\n",
2806  solver->parameters.algorithm ? "Q" : "SARSA",
2807  solver->parameters.alpha,
2808  solver->parameters.beta,
2809  solver->parameters.lambda);
2810  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() exploration_ratio = %f, temperature = %f, ActionSelection = %s\n",
2811  solver->parameters.epsilon,
2812  solver->parameters.temperature,
2813  solver->parameters.select ? "EGREEDY" : "SOFTMAX");
2814  LOG(GNUNET_ERROR_TYPE_DEBUG, "init() RBF_DIVISOR = %llu\n",
2815  solver->parameters.rbf_divisor);
2816 
2817  return &sf;
2818 }
GAS_solver_stop_get_preferred_address s_get_stop
Tell solver stop notifying ATS about changes for this peers.
static void GAS_ril_address_change_preference(void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
Change relative preference for quality in solver.
static void GAS_ril_address_delete(void *solver, struct ATS_Address *address)
Delete an address in the solver.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
int GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
GAS_solver_address_add s_add
Add a new address for a peer to the solver.
int GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
const char * GNUNET_NT_to_string(enum GNUNET_NetworkType net)
Convert a enum GNUNET_NetworkType to a string.
Definition: nt.c:43
#define RIL_DEFAULT_SELECT
double lambda
Trace-decay factor for eligibility traces.
struct GNUNET_STATISTICS_Handle * stats
Statistics handle to be used by the solver.
#define RIL_DEFAULT_DISCOUNT_GAMMA
#define RIL_DEFAULT_EXPLORE_DECAY
#define RIL_DEFAULT_RBF_DIVISOR
static void GAS_ril_address_add(void *solver, struct ATS_Address *address, uint32_t network)
Add a new address for a peer to the solver.
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
static void GAS_ril_bulk_stop(void *solver)
Bulk operation done.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void GAS_ril_stop_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver stop notifying ATS about changes for this peers.
A handle for the reinforcement learning solver.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define RIL_DEFAULT_GRADIENT_STEP_SIZE
GAS_solver_address_feedback_preference s_feedback
Give feedback about the current assignment.
int GNUNET_CONFIGURATION_get_value_float(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, float *number)
Get a configuration value that should be a floating point number.
GAS_solver_address_property_changed s_address_update_property
Update the properties of an address in the solver.
double temperature_decay
Decay factor of the temperature value.
#define RIL_DEFAULT_WELFARE
#define GNUNET_NO
Definition: gnunet_common.h:78
struct RIL_Scope * network_entries
Array of networks with global assignment state.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_new(type)
Allocate a struct or union of the given type.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void GAS_ril_address_property_changed(void *solver, struct ATS_Address *address)
Update the properties of an address in the solver.
unsigned long long step_count
Number of performed steps.
double temperature
Softmax action-selection temperature.
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
#define RIL_DEFAULT_TRACE_DECAY
unsigned long long simulate
Simulate steps, i.e.
void GNUNET_STRINGS_utf8_toupper(const char *input, char *output)
Convert the utf-8 input string to upper case.
Definition: strings.c:577
static void GAS_ril_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
#define RIL_DEFAULT_ALGORITHM
enum RIL_Select select
Action selection strategy;.
enum GNUNET_NetworkType type
ATS network type.
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.
double beta
Learning discount variable in the TD-update for semi-MDPs.
GAS_solver_address_delete s_del
Delete an address in the solver.
#define RIL_DEFAULT_TEMPERATURE_DECAY
enum RIL_Algorithm algorithm
The TD-algorithm to use.
#define RIL_DEFAULT_STEP_TIME_MAX
struct RIL_Learning_Parameters parameters
Learning parameters.
GAS_solver_address_change_preference s_pref
Change relative preference for quality in solver.
#define RIL_DEFAULT_TEMPERATURE
int GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
#define RIL_DEFAULT_EXPLORE_RATIO
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
unsigned int networks_count
Networks count.
double epsilon_decay
Decay factor of the explore ratio.
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
void * cls
Closure to pass to all solver functions in this struct.
#define RIL_DEFAULT_STEP_TIME_MIN
double epsilon_init
Initial exploration ratio value.
GAS_solver_get_preferred_address s_get
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
unsigned long long in_quota[6]
Array of configured inbound quotas Order according to networks in network array.
#define RIL_DEFAULT_DISCOUNT_BETA
uint32_t bw_out_available
Total available outbound bandwidth.
GAS_solver_bulk_start s_bulk_start
Start a bulk operation.
int done
Shutdown.
static void GAS_ril_address_preference_feedback(void *solver, struct GNUNET_SERVICE_Client *application, const struct GNUNET_PeerIdentity *peer, const struct GNUNET_TIME_Relative scope, enum GNUNET_ATS_PreferenceKind kind, double score)
Give feedback about the current assignment.